From 3afa9f00b9c644e1501a2e642112ec62bb414cd9 Mon Sep 17 00:00:00 2001 From: LoadingALIAS Date: Sun, 16 Aug 2026 22:55:30 -0400 Subject: [PATCH 01/12] crypto: enforce lint-clean safety boundaries Remove repository-wide lint suppression debt and make feature and target reachability explicit across portable and accelerated implementations. Harden arithmetic, bounds, secret handling, unsafe and ASM contracts, dispatch, tests, fuzz targets, and validation tooling. This intentionally breaks RSA blinding inputs, forced ChaCha20 diagnostic calls, and RISC-V CRC-64 force modes. --- .cargo/config.toml | 65 - .changes/add-debug-output-for-4cec.md | 5 + .changes/add-debug-output-for-6a2e.md | 5 + .changes/add-redacted-debug-output-4c0e.md | 5 + .changes/document-checksum-and-digest-e536.md | 5 + .../document-shake128-and-shake256-2928.md | 5 + .changes/harden-aegis256-aesni-a37b.md | 5 + .../harden-blake2b-length-handling-4ab7.md | 5 + .../harden-blake2s-length-handling-454e.md | 5 + .../harden-blake3-tree-arithmetic-32c1.md | 5 + .changes/harden-ecdsa-limb-boundaries-6503.md | 5 + .../harden-ed25519-simd-boundaries-de52.md | 5 + .changes/harden-ml-kem-encoding-2483.md | 5 + .changes/harden-pbkdf2-block-indexing-54fe.md | 5 + .../remove-unused-riscv-crc64-surfac-b7f9.md | 5 + .../report-unavailable-argon2-backe-8b41.md | 5 + .../unsafe-chacha20-diagnostic-backe-51d5.md | 5 + .config/ci-tool-archives.tsv | 8 +- .config/nextest.toml | 4 + .config/rail.toml | 16 +- .config/target-matrix.json | 22 +- .config/toolchains.toml | 1 + .github/actions/setup-toolchain/action.yaml | 20 +- .github/actions/setup/action.yaml | 9 +- .github/workflows/_ci-suite.yaml | 18 + .github/workflows/_rust-job.yaml | 5 + .github/workflows/bench.yaml | 1 + .github/workflows/ci.yaml | 2 +- .github/workflows/ct.yaml | 1 + .github/workflows/riscv.yaml | 2 + .github/workflows/rsa.yaml | 1 + .github/workflows/weekly.yaml | 4 + .gitignore | 4 +- .zed/settings.json | 13 + .zed/tasks.json | 18 + Cargo.lock | 510 ++--- Cargo.toml | 130 +- README.md | 11 +- benches/aead.rs | 231 ++- benches/aead_kernels.rs | 20 +- benches/ascon.rs | 7 +- benches/auth.rs | 276 ++- benches/blake2.rs | 58 +- benches/blake3.rs | 54 +- benches/common/mod.rs | 6 +- benches/password_hashing.rs | 139 +- benches/rsa.rs | 177 +- benches/sha2.rs | 2 +- clippy.toml | 2 + deny.toml | 19 +- docs/features.md | 16 +- docs/migration/README.md | 18 +- docs/migration/RustCrypto/aes-gcm-siv.md | 18 +- docs/migration/RustCrypto/aes-gcm.md | 16 +- docs/migration/RustCrypto/argon2.md | 6 +- docs/migration/RustCrypto/ascon-aead.md | 16 +- docs/migration/RustCrypto/ascon-hash.md | 8 +- docs/migration/RustCrypto/blake2.md | 8 +- docs/migration/RustCrypto/chacha20poly1305.md | 24 +- docs/migration/RustCrypto/ed25519-dalek.md | 12 +- docs/migration/RustCrypto/hkdf.md | 8 +- docs/migration/RustCrypto/hmac.md | 8 +- docs/migration/RustCrypto/p256.md | 6 +- docs/migration/RustCrypto/p384.md | 10 +- docs/migration/RustCrypto/pbkdf2.md | 8 +- docs/migration/RustCrypto/rsa.md | 4 +- docs/migration/RustCrypto/scrypt.md | 4 +- docs/migration/RustCrypto/sha2.md | 8 +- docs/migration/RustCrypto/sha3.md | 8 +- docs/migration/RustCrypto/x25519-dalek.md | 10 +- docs/migration/aegis.md | 8 +- docs/migration/api-boundaries.md | 22 + docs/migration/aws-lc-rs.md | 2 +- docs/migration/blake3.md | 8 +- docs/migration/crc-fast.md | 8 +- docs/migration/crc.md | 8 +- docs/migration/crc32c.md | 6 +- docs/migration/crc32fast.md | 6 +- docs/migration/crc64fast.md | 10 +- docs/migration/dryoc.md | 2 +- docs/migration/openssl.md | 2 +- docs/migration/rapidhash.md | 2 +- docs/migration/ring.md | 2 +- docs/migration/sha3-kmac.md | 6 +- docs/migration/tiny-keccak.md | 8 +- docs/migration/twox-hash.md | 6 +- docs/migration/xxhash-rust.md | 8 +- docs/secret-ownership.md | 3 +- docs/test-vector-coverage.md | 2 +- docs/types.md | 2 +- examples/aead_seal_open.rs | 4 +- examples/basic.rs | 48 +- examples/mlkem_encapsulation.rs | 4 +- examples/parallel.rs | 19 +- examples/password_hashing.rs | 12 +- fuzz-packages/aead-aegis256/Cargo.lock | 10 +- fuzz-packages/aead-aes128gcm/Cargo.lock | 10 +- fuzz-packages/aead-aes128gcmsiv/Cargo.lock | 129 +- fuzz-packages/aead-aes128gcmsiv/Cargo.toml | 2 +- fuzz-packages/aead-aes256gcm/Cargo.lock | 10 +- fuzz-packages/aead-aes256gcmsiv/Cargo.lock | 129 +- fuzz-packages/aead-aes256gcmsiv/Cargo.toml | 2 +- fuzz-packages/aead-ascon128/Cargo.lock | 10 +- .../aead-chacha20poly1305/Cargo.lock | 10 +- fuzz-packages/aead-nonce-counter/Cargo.lock | 10 +- .../aead-xchacha20poly1305/Cargo.lock | 10 +- fuzz-packages/auth-argon2/Cargo.lock | 109 +- fuzz-packages/auth-argon2/Cargo.toml | 2 +- fuzz-packages/auth-ed25519/Cargo.lock | 193 +- fuzz-packages/auth-ed25519/Cargo.toml | 2 +- fuzz-packages/auth-hkdf/Cargo.lock | 10 +- fuzz-packages/auth-hmac/Cargo.lock | 10 +- fuzz-packages/auth-kmac256/Cargo.lock | 10 +- fuzz-packages/auth-pbkdf2/Cargo.lock | 10 +- fuzz-packages/auth-phc/Cargo.lock | 10 +- fuzz-packages/auth-scrypt/Cargo.lock | 10 +- fuzz-packages/auth-x25519/Cargo.lock | 88 +- fuzz-packages/auth-x25519/Cargo.toml | 2 +- fuzz-packages/checksums-crc16/Cargo.lock | 10 +- fuzz-packages/checksums-crc24/Cargo.lock | 10 +- fuzz-packages/checksums-crc32/Cargo.lock | 10 +- fuzz-packages/checksums-crc64/Cargo.lock | 10 +- fuzz-packages/fast-rapidhash/Cargo.lock | 10 +- fuzz-packages/fast-xxh3/Cargo.lock | 10 +- fuzz-packages/hash-ascon/Cargo.lock | 10 +- fuzz-packages/hash-blake2/Cargo.lock | 76 +- fuzz-packages/hash-blake2/Cargo.toml | 2 +- fuzz-packages/hash-blake3/Cargo.lock | 14 +- fuzz-packages/hash-sha2/Cargo.lock | 10 +- fuzz-packages/hash-sha3/Cargo.lock | 14 +- fuzz-packages/surface-hex-parse/Cargo.lock | 10 +- fuzz-packages/traits-io/Cargo.lock | 10 +- fuzz/Cargo.lock | 507 +---- fuzz/Cargo.toml | 14 +- fuzz/fuzz_targets/aead_aegis256.rs | 2 +- fuzz/fuzz_targets/aead_aes128gcm.rs | 2 +- fuzz/fuzz_targets/aead_aes128gcmsiv.rs | 2 +- fuzz/fuzz_targets/aead_aes256gcm.rs | 2 +- fuzz/fuzz_targets/aead_aes256gcmsiv.rs | 2 +- fuzz/fuzz_targets/aead_ascon128.rs | 2 +- fuzz/fuzz_targets/aead_chacha20poly1305.rs | 2 +- fuzz/fuzz_targets/aead_nonce_counter.rs | 2 +- fuzz/fuzz_targets/aead_xchacha20poly1305.rs | 2 +- fuzz/fuzz_targets/auth_argon2d.rs | 2 +- fuzz/fuzz_targets/auth_argon2i.rs | 2 +- fuzz/fuzz_targets/auth_argon2id.rs | 2 +- fuzz/fuzz_targets/auth_ed25519.rs | 2 +- fuzz/fuzz_targets/auth_ed25519_verify.rs | 2 +- fuzz/fuzz_targets/auth_hkdf_sha256.rs | 2 +- fuzz/fuzz_targets/auth_hkdf_sha384.rs | 2 +- fuzz/fuzz_targets/auth_hmac_sha256.rs | 2 +- fuzz/fuzz_targets/auth_hmac_sha384.rs | 2 +- fuzz/fuzz_targets/auth_hmac_sha512.rs | 2 +- fuzz/fuzz_targets/auth_kmac256.rs | 2 +- fuzz/fuzz_targets/auth_pbkdf2.rs | 2 +- fuzz/fuzz_targets/auth_phc.rs | 2 +- fuzz/fuzz_targets/auth_scrypt.rs | 2 +- fuzz/fuzz_targets/auth_x25519.rs | 2 +- fuzz/fuzz_targets/checksum_crc.rs | 2 +- fuzz/fuzz_targets/fast_rapidhash.rs | 2 +- fuzz/fuzz_targets/fast_xxh3.rs | 2 +- fuzz/fuzz_targets/hash_ascon.rs | 2 +- fuzz/fuzz_targets/hash_ascon_cxof.rs | 2 +- fuzz/fuzz_targets/hash_blake2b.rs | 2 +- fuzz/fuzz_targets/hash_blake2s.rs | 2 +- fuzz/fuzz_targets/hash_blake3.rs | 2 +- fuzz/fuzz_targets/hash_blake3_derive.rs | 2 +- fuzz/fuzz_targets/hash_blake3_keyed.rs | 2 +- fuzz/fuzz_targets/hash_cshake256.rs | 2 +- fuzz/fuzz_targets/hash_sha2.rs | 2 +- fuzz/fuzz_targets/hash_sha3.rs | 2 +- fuzz/fuzz_targets/hex_parse.rs | 2 +- fuzz/fuzz_targets/traits_io.rs | 2 +- fuzz/support/src/lib.rs | 18 +- fuzz/target_impls/aead_aegis256.rs | 22 +- fuzz/target_impls/aead_aes128gcm.rs | 16 +- fuzz/target_impls/aead_aes128gcmsiv.rs | 18 +- fuzz/target_impls/aead_aes256gcm.rs | 16 +- fuzz/target_impls/aead_aes256gcmsiv.rs | 18 +- fuzz/target_impls/aead_ascon128.rs | 16 +- fuzz/target_impls/aead_chacha20poly1305.rs | 17 +- fuzz/target_impls/aead_nonce_counter.rs | 12 +- fuzz/target_impls/aead_xchacha20poly1305.rs | 17 +- fuzz/target_impls/auth_argon2d.rs | 5 +- fuzz/target_impls/auth_argon2i.rs | 5 +- fuzz/target_impls/auth_argon2id.rs | 5 +- fuzz/target_impls/auth_ecdsa_sign.rs | 30 +- fuzz/target_impls/auth_ecdsa_verify.rs | 2 +- fuzz/target_impls/auth_ed25519.rs | 2 +- fuzz/target_impls/auth_ed25519_verify.rs | 2 +- fuzz/target_impls/auth_hkdf_sha256.rs | 4 +- fuzz/target_impls/auth_hkdf_sha384.rs | 4 +- fuzz/target_impls/auth_hmac_sha256.rs | 4 +- fuzz/target_impls/auth_hmac_sha384.rs | 4 +- fuzz/target_impls/auth_hmac_sha512.rs | 4 +- fuzz/target_impls/auth_kmac256.rs | 10 +- fuzz/target_impls/auth_mlkem1024.rs | 10 +- fuzz/target_impls/auth_mlkem512.rs | 10 +- fuzz/target_impls/auth_mlkem768.rs | 10 +- fuzz/target_impls/auth_pbkdf2.rs | 30 +- fuzz/target_impls/auth_phc.rs | 22 +- fuzz/target_impls/auth_rsa_import.rs | 39 +- fuzz/target_impls/auth_rsa_private_ops.rs | 81 +- fuzz/target_impls/auth_rsa_protocol.rs | 50 +- fuzz/target_impls/auth_rsa_public_key.rs | 44 +- fuzz/target_impls/auth_rsa_verify.rs | 17 +- fuzz/target_impls/auth_scrypt.rs | 5 +- fuzz/target_impls/auth_x25519.rs | 8 +- fuzz/target_impls/checksum_crc.rs | 2 +- fuzz/target_impls/checksum_crc16.rs | 2 +- fuzz/target_impls/checksum_crc24.rs | 2 +- fuzz/target_impls/checksum_crc32.rs | 2 +- fuzz/target_impls/checksum_crc64.rs | 2 +- fuzz/target_impls/fast_rapidhash.rs | 2 +- fuzz/target_impls/fast_xxh3.rs | 2 +- fuzz/target_impls/hash_ascon.rs | 2 +- fuzz/target_impls/hash_ascon_cxof.rs | 2 +- fuzz/target_impls/hash_blake2b.rs | 21 +- fuzz/target_impls/hash_blake2s.rs | 21 +- fuzz/target_impls/hash_blake3.rs | 2 +- fuzz/target_impls/hash_blake3_derive.rs | 2 +- fuzz/target_impls/hash_blake3_keyed.rs | 2 +- fuzz/target_impls/hash_cshake256.rs | 2 +- fuzz/target_impls/hash_sha2.rs | 2 +- fuzz/target_impls/hash_sha3.rs | 2 +- fuzz/target_impls/hex_parse.rs | 49 +- fuzz/target_impls/traits_io.rs | 14 +- fuzz/tests/corpus_replay.rs | 254 ++- justfile | 115 +- rust-toolchain.toml | 28 +- rustfmt.toml | 19 +- scripts/README.md | 9 +- scripts/check/check-all.sh | 26 +- scripts/check/check-ibm.sh | 7 +- scripts/check/check-linux.sh | 13 +- scripts/check/check-win.sh | 2 +- scripts/check/check.sh | 12 +- scripts/check/lint-independent-workspaces.sh | 129 ++ scripts/check/rsa-asm-provenance.sh | 6 +- scripts/ci/changed-test-planning-test.sh | 11 +- scripts/ci/check-ci-ownership-test.sh | 2 +- scripts/ci/check-ci-ownership.sh | 2 +- scripts/ci/check-locked-cargo-test.sh | 37 + scripts/ci/check-locked-cargo.sh | 83 + scripts/ci/check-worktree-test.sh | 7 +- scripts/ci/ci-check.sh | 5 +- scripts/ci/cross-targets.sh | 6 +- scripts/ci/install-tools.sh | 8 +- scripts/ci/mlkem-aarch64-gate.sh | 12 +- scripts/ci/native-check.sh | 8 +- scripts/ci/nostd-wasm-suite.sh | 20 +- scripts/ci/release-identity-test.sh | 2 +- scripts/ci/run-bench.sh | 4 +- scripts/ci/run-rust-job-test.sh | 2 +- scripts/ci/run-rust-job.sh | 18 +- scripts/ci/tool-integrity-test.sh | 25 +- scripts/ct/dudect.sh | 4 +- scripts/lib/common.sh | 22 +- scripts/lib/fuzz-packages.sh | 3 + scripts/lib/target-matrix.sh | 60 +- scripts/lib/toolchain.sh | 36 +- scripts/test/test-coverage.sh | 6 +- scripts/test/test-fuzz-asan.sh | 2 + scripts/test/test-fuzz-scheduler-test.sh | 7 +- scripts/test/test-fuzz.sh | 1 + scripts/test/test-miri.sh | 11 +- scripts/test/test-rsa-leakage.sh | 2 +- scripts/test/test.sh | 20 +- src/aead/aegis256.rs | 260 +-- src/aead/aegis256/aarch64_ce.rs | 355 ++-- src/aead/aegis256/powerpc64_ppc.rs | 376 ++-- src/aead/aegis256/riscv64_vperm.rs | 368 ---- src/aead/aegis256/s390x_vperm.rs | 144 +- src/aead/aegis256/x86_64_ni.rs | 336 ++-- src/aead/aes.rs | 1782 +++-------------- src/aead/aes/aarch64/asm.rs | 6 +- src/aead/aes/aarch64_ce.rs | 141 +- src/aead/aes/riscv64_fixslice_aes.rs | 26 +- src/aead/aes/riscv64_vperm_aes.rs | 617 ------ src/aead/aes/x86_64/asm.rs | 62 +- .../asm/rscrypto_aes_gcm_x86_64_linux.s | 320 --- src/aead/aes/x86_64_ni.rs | 106 - src/aead/aes128gcm.rs | 185 +- src/aead/aes128gcmsiv.rs | 251 ++- src/aead/aes256gcm.rs | 193 +- src/aead/aes256gcmsiv.rs | 240 ++- src/aead/aes_round.rs | 29 +- src/aead/ascon128.rs | 163 +- src/aead/chacha20.rs | 407 ++-- src/aead/chacha20/aarch64_neon.rs | 60 +- src/aead/chacha20/powerpc64_vsx.rs | 49 +- src/aead/chacha20/riscv64_vector.rs | 27 +- src/aead/chacha20/s390x_vector.rs | 49 +- src/aead/chacha20/wasm32_simd128.rs | 74 +- src/aead/chacha20/x86_64_avx2.rs | 83 +- src/aead/chacha20/x86_64_avx512.rs | 101 +- src/aead/chacha20/x86_64_ssse3_x4.rs | 67 +- src/aead/chacha20poly1305.rs | 132 +- src/aead/chacha20poly1305/aarch64_asm.rs | 22 +- src/aead/chacha20poly1305/x86_64_asm.rs | 36 +- src/aead/ghash.rs | 42 +- src/aead/introspect.rs | 11 +- src/aead/mod.rs | 239 ++- src/aead/nonce_counter.rs | 123 +- src/aead/poly1305.rs | 807 +++++--- src/aead/poly1305/aarch64_neon.rs | 179 +- src/aead/poly1305/powerpc64_vsx.rs | 19 +- src/aead/poly1305/riscv64_vector.rs | 133 +- src/aead/poly1305/s390x_vector.rs | 19 +- src/aead/poly1305/x86_64_avx2_par4.rs | 897 +++++---- src/aead/polyval.rs | 642 +++--- src/aead/targets.rs | 269 ++- src/aead/xchacha20poly1305.rs | 40 +- src/auth/argon2/aarch64.rs | 246 +-- src/auth/argon2/dispatch.rs | 7 +- src/auth/argon2/kernels.rs | 35 +- src/auth/argon2/mod.rs | 565 +++--- src/auth/argon2/power.rs | 130 +- src/auth/argon2/riscv64.rs | 345 ++-- src/auth/argon2/s390x.rs | 274 +-- src/auth/argon2/wasm.rs | 293 ++- src/auth/argon2/x86_64.rs | 95 +- src/auth/curve25519_edwards.rs | 33 +- src/auth/ecdsa.rs | 417 ++-- src/auth/ecdsa_aarch64_asm.rs | 2 - src/auth/ecdsa_p384_field.rs | 389 ++-- src/auth/ecdsa_x86_64_asm.rs | 2 - src/auth/ed25519.rs | 48 +- src/auth/ed25519/aarch64_asm.rs | 2 - src/auth/ed25519/basepoint_table_ifma.rs | 4 +- src/auth/ed25519/field_avx2.rs | 307 ++- src/auth/ed25519/field_ifma.rs | 156 +- src/auth/ed25519/point.rs | 177 +- src/auth/ed25519/point_avx2.rs | 317 +-- src/auth/ed25519/scalar.rs | 220 +- src/auth/ed25519/x86_64_asm.rs | 2 - src/auth/hkdf.rs | 49 +- src/auth/hmac.rs | 48 +- src/auth/hmac_sha3.rs | 2 - src/auth/kmac.rs | 25 +- src/auth/mlkem.rs | 2 +- src/auth/mlkem/portable.rs | 1374 +++++++------ src/auth/mlkem/portable/aarch64.rs | 22 +- src/auth/mlkem/portable/x86_64.rs | 268 ++- src/auth/mod.rs | 45 +- src/auth/pbkdf2.rs | 260 +-- src/auth/phc.rs | 271 +-- src/auth/poly1305.rs | 152 +- src/auth/rsa.rs | 1345 +++++++------ src/auth/rsa_aarch64_asm.rs | 28 +- src/auth/rsa_aarch64_linux_asm.rs | 28 +- src/auth/rsa_x86_64_asm.rs | 24 +- src/auth/scrypt.rs | 186 +- src/auth/x25519.rs | 23 +- src/auth/x25519/aarch64_asm.rs | 2 - src/auth/x25519/x86_64_asm.rs | 2 - src/backend/ascon.rs | 3 +- src/backend/cache.rs | 55 +- src/backend/curve25519.rs | 50 +- src/backend/mod.rs | 27 +- src/checksum/common/clmul.rs | 29 +- src/checksum/common/combine.rs | 65 +- src/checksum/common/kernels.rs | 6 +- src/checksum/common/mod.rs | 50 +- src/checksum/common/portable.rs | 135 +- src/checksum/common/prefetch.rs | 45 +- src/checksum/common/reference.rs | 26 +- src/checksum/common/tables.rs | 137 +- src/checksum/common/tests.rs | 50 +- src/checksum/crc16/aarch64.rs | 481 +++-- src/checksum/crc16/config.rs | 14 +- src/checksum/crc16/kernels.rs | 77 +- src/checksum/crc16/keys.rs | 73 +- src/checksum/crc16/mod.rs | 34 +- src/checksum/crc16/portable.rs | 9 +- src/checksum/crc16/power.rs | 104 +- src/checksum/crc16/riscv64.rs | 939 ++++----- src/checksum/crc16/s390x.rs | 488 +++-- src/checksum/crc16/x86_64.rs | 390 +++- src/checksum/crc24/aarch64.rs | 176 +- src/checksum/crc24/config.rs | 12 +- src/checksum/crc24/kernels.rs | 51 +- src/checksum/crc24/keys.rs | 71 +- src/checksum/crc24/mod.rs | 34 +- src/checksum/crc24/portable.rs | 5 +- src/checksum/crc24/power.rs | 80 +- src/checksum/crc24/reflected.rs | 12 +- src/checksum/crc24/riscv64.rs | 897 ++++----- src/checksum/crc24/s390x.rs | 464 ++--- src/checksum/crc24/x86_64.rs | 351 +++- src/checksum/crc32/aarch64.rs | 801 +++++--- src/checksum/crc32/clmul.rs | 24 +- src/checksum/crc32/config.rs | 12 +- src/checksum/crc32/kernels.rs | 108 +- src/checksum/crc32/mod.rs | 64 +- src/checksum/crc32/portable.rs | 13 +- src/checksum/crc32/power.rs | 101 +- src/checksum/crc32/riscv64.rs | 949 +++++---- src/checksum/crc32/s390x.rs | 501 +++-- src/checksum/crc32/x86_64.rs | 901 ++++++--- src/checksum/crc64/aarch64.rs | 225 ++- src/checksum/crc64/config.rs | 42 +- src/checksum/crc64/kernels.rs | 126 +- src/checksum/crc64/mod.rs | 51 +- src/checksum/crc64/portable.rs | 19 +- src/checksum/crc64/power.rs | 97 +- src/checksum/crc64/riscv64.rs | 1547 -------------- src/checksum/crc64/s390x.rs | 570 +++--- src/checksum/crc64/x86_64.rs | 753 ++++--- src/checksum/diag.rs | 55 + src/checksum/dispatchers.rs | 8 +- src/checksum/kernel_table.rs | 285 ++- src/checksum/macros.rs | 7 +- src/checksum/mod.rs | 7 +- src/hashes/common/mod.rs | 2 +- src/hashes/common/prefetch.rs | 1 - src/hashes/crypto/ascon.rs | 137 +- src/hashes/crypto/ascon/aarch64.rs | 8 +- src/hashes/crypto/ascon/dispatch.rs | 34 +- src/hashes/crypto/ascon/dispatch_tables.rs | 56 +- src/hashes/crypto/ascon/kernel_test.rs | 30 +- src/hashes/crypto/ascon/kernels.rs | 15 +- src/hashes/crypto/ascon/x86_64_avx2.rs | 106 +- src/hashes/crypto/ascon/x86_64_avx512.rs | 106 +- src/hashes/crypto/blake2b/kernels.rs | 49 +- src/hashes/crypto/blake2b/mod.rs | 274 ++- src/hashes/crypto/blake2b/riscv64.rs | 148 +- src/hashes/crypto/blake2b/wasm.rs | 173 +- src/hashes/crypto/blake2b/x86_64.rs | 106 +- src/hashes/crypto/blake2s/kernels.rs | 42 +- src/hashes/crypto/blake2s/mod.rs | 152 +- src/hashes/crypto/blake2s/riscv64.rs | 2 - src/hashes/crypto/blake2s/wasm.rs | 38 +- src/hashes/crypto/blake2s/x86_64.rs | 32 +- src/hashes/crypto/blake3/aarch64.rs | 491 +++-- src/hashes/crypto/blake3/aarch64/asm.rs | 18 +- src/hashes/crypto/blake3/control.rs | 19 +- src/hashes/crypto/blake3/dispatch.rs | 23 +- src/hashes/crypto/blake3/dispatch_tables.rs | 98 +- src/hashes/crypto/blake3/kernel_test.rs | 18 +- src/hashes/crypto/blake3/kernels.rs | 1184 +++++++---- src/hashes/crypto/blake3/mod.rs | 1019 +++++----- src/hashes/crypto/blake3/parallel.rs | 70 +- src/hashes/crypto/blake3/x86_64.rs | 208 +- src/hashes/crypto/blake3/x86_64/asm.rs | 116 +- ...rscrypto_blake3_avx2_x86-64_apple_darwin.s | 1 - ...crypto_blake3_avx512_x86-64_apple_darwin.s | 1 - src/hashes/crypto/blake3/x86_64/avx2.rs | 279 ++- src/hashes/crypto/blake3/x86_64/avx512.rs | 249 +-- src/hashes/crypto/blake3/x86_64/sse41.rs | 111 +- src/hashes/crypto/cshake.rs | 4 +- src/hashes/crypto/dispatch_util.rs | 12 +- src/hashes/crypto/keccak.rs | 164 +- src/hashes/crypto/keccak/aarch64.rs | 37 +- src/hashes/crypto/keccak/dispatch.rs | 4 +- src/hashes/crypto/keccak/dispatch_tables.rs | 16 +- src/hashes/crypto/keccak/kernel_test.rs | 31 +- src/hashes/crypto/keccak/kernels.rs | 10 +- src/hashes/crypto/keccak/s390x.rs | 2 - src/hashes/crypto/keccak/x86_64.rs | 44 +- src/hashes/crypto/mod.rs | 4 - src/hashes/crypto/sha224.rs | 10 +- src/hashes/crypto/sha224/dispatch_tables.rs | 21 +- src/hashes/crypto/sha224/kernel_test.rs | 48 +- src/hashes/crypto/sha224/kernels.rs | 8 +- src/hashes/crypto/sha256/aarch64.rs | 9 +- src/hashes/crypto/sha256/dispatch_tables.rs | 21 +- src/hashes/crypto/sha256/kernel_test.rs | 48 +- src/hashes/crypto/sha256/kernels.rs | 34 +- src/hashes/crypto/sha256/mod.rs | 24 +- src/hashes/crypto/sha256/riscv64.rs | 15 +- src/hashes/crypto/sha256/s390x.rs | 3 - src/hashes/crypto/sha256/wasm.rs | 55 +- src/hashes/crypto/sha256/x86_64.rs | 3 - src/hashes/crypto/sha3.rs | 102 +- src/hashes/crypto/sha384.rs | 9 +- src/hashes/crypto/sha384/dispatch_tables.rs | 27 +- src/hashes/crypto/sha384/kernel_test.rs | 44 +- src/hashes/crypto/sha384/kernels.rs | 8 +- src/hashes/crypto/sha512/aarch64.rs | 18 +- src/hashes/crypto/sha512/dispatch_tables.rs | 31 +- src/hashes/crypto/sha512/kernel_test.rs | 55 +- src/hashes/crypto/sha512/kernels.rs | 8 +- src/hashes/crypto/sha512/mod.rs | 22 +- src/hashes/crypto/sha512/riscv64.rs | 15 +- src/hashes/crypto/sha512/s390x.rs | 3 - src/hashes/crypto/sha512/wasm.rs | 36 +- src/hashes/crypto/sha512/x86_64.rs | 9 +- src/hashes/crypto/sha512/x86_64_avx2.rs | 52 +- src/hashes/crypto/sha512/x86_64_avx512vl.rs | 64 +- src/hashes/crypto/sha512_256.rs | 8 +- .../crypto/sha512_256/dispatch_tables.rs | 27 +- src/hashes/crypto/sha512_256/kernel_test.rs | 44 +- src/hashes/crypto/sha512_256/kernels.rs | 8 +- src/hashes/crypto/sp800185.rs | 11 +- src/hashes/fast/rapidhash.rs | 46 +- src/hashes/fast/rapidhash/stream.rs | 95 +- src/hashes/fast/xxh3.rs | 289 ++- src/hashes/fast/xxh3/aarch64_neon.rs | 47 +- src/hashes/fast/xxh3/dispatch.rs | 293 +-- src/hashes/fast/xxh3/dispatch_tables.rs | 18 +- src/hashes/fast/xxh3/kernels.rs | 32 +- src/hashes/fast/xxh3/power.rs | 66 +- src/hashes/fast/xxh3/s390x.rs | 81 +- src/hashes/fast/xxh3/stream.rs | 16 +- src/hashes/fast/xxh3/x86_64_avx2.rs | 71 +- src/hashes/fast/xxh3/x86_64_avx512.rs | 67 +- src/hashes/mod.rs | 8 +- src/hashes/util.rs | 6 +- src/hex.rs | 279 ++- src/lib.rs | 81 +- src/macros.rs | 47 +- src/platform/caps.rs | 241 ++- src/platform/detect.rs | 35 +- src/platform/detect/arch/aarch64.rs | 186 +- src/platform/detect/arch/power.rs | 7 +- src/platform/detect/arch/riscv.rs | 8 +- src/platform/detect/arch/s390x.rs | 19 +- src/platform/detect/arch/x86.rs | 107 +- src/platform/detect/cache_override.rs | 8 +- src/platform/detect/compile_time.rs | 94 +- src/platform/detect/tests.rs | 82 +- src/platform/mod.rs | 29 + src/platform/target_matrix.rs | 8 +- src/secret.rs | 8 +- src/traits/aead.rs | 1 + src/traits/ct.rs | 148 +- src/traits/error.rs | 11 +- src/traits/io.rs | 78 +- tests/aead_foundations.rs | 68 +- tests/aead_kernel_equivalence.rs | 16 +- tests/aead_wycheproof.rs | 269 ++- tests/aegis256_oracle.rs | 41 +- tests/aes128gcm_oracle.rs | 65 +- tests/aes128gcmsiv_oracle.rs | 64 +- tests/aes256gcm_oracle.rs | 65 +- tests/aes256gcmsiv_oracle.rs | 64 +- tests/aes_gcm_aarch64_asm_oracle.rs | 36 +- tests/api_consistency.rs | 179 +- tests/argon2_differential.rs | 61 +- tests/argon2_kernels.rs | 47 +- tests/argon2_miri.rs | 8 +- tests/argon2_parallel.rs | 53 +- tests/argon2_vectors.rs | 15 +- tests/ascon_aead_oracle.rs | 41 +- tests/ascon_cxof_vectors.rs | 7 +- tests/ascon_differential.rs | 8 +- tests/ascon_final_kats.rs | 25 +- tests/ascon_hash_oracle.rs | 4 +- tests/ascon_official_vectors.rs | 12 +- tests/blake2_differential.rs | 122 +- tests/blake2_official_vectors.rs | 19 +- tests/blake3_differential.rs | 8 +- tests/blake3_official_vectors.rs | 16 +- tests/chacha20poly1305.rs | 84 +- tests/common/array.rs | 10 + tests/common/mod.rs | 28 +- tests/common_properties.rs | 36 +- tests/crc16_properties.rs | 28 +- tests/crc32_properties.rs | 26 +- tests/cshake256_differential.rs | 24 +- tests/cshake256_nist_vectors.rs | 4 +- tests/ecdsa_oracle.rs | 83 +- tests/ecdsa_wycheproof.rs | 20 +- tests/ed25519_oracle.rs | 22 +- tests/ed25519_rfc8032_vectors.rs | 5 +- tests/ed25519_wycheproof.rs | 41 +- tests/fast_hash_allocations.rs | 24 +- tests/getrandom_smoke.rs | 34 +- tests/hkdf_sha256_vectors.rs | 16 +- tests/hkdf_sha384_vectors.rs | 35 +- tests/hkdf_sha512_vectors.rs | 30 +- tests/hkdf_wycheproof.rs | 44 +- tests/hmac_sha256_proptest.rs | 3 +- tests/hmac_sha256_vectors.rs | 20 +- tests/hmac_sha2_family_vectors.rs | 18 +- tests/hmac_sha384_proptest.rs | 3 +- tests/hmac_sha3_vectors.rs | 9 +- tests/hmac_sha512_proptest.rs | 3 +- tests/hmac_wycheproof.rs | 51 +- tests/kmac128_differential.rs | 36 +- tests/kmac128_nist_vectors.rs | 22 +- tests/kmac256_differential.rs | 36 +- tests/kmac256_nist_vectors.rs | 28 +- tests/kmac_wycheproof.rs | 48 +- tests/migration_aws_lc_rs.rs | 77 +- tests/migration_docs.rs | 17 +- tests/migration_dryoc.rs | 37 +- tests/migration_ring.rs | 46 +- tests/mlkem_acvp.rs | 103 +- tests/mlkem_ops.rs | 3 +- tests/mlkem_properties.rs | 82 +- tests/owned_equality.rs | 1 + tests/pbkdf2_differential.rs | 39 +- tests/pbkdf2_kat_vectors.rs | 67 +- tests/pbkdf2_wycheproof.rs | 53 +- tests/phc_external_entropy.rs | 20 +- tests/phc_roundtrip.rs | 113 +- tests/platform_amx_permission.rs | 2 - tests/platform_override_race.rs | 25 +- tests/platform_override_validation.rs | 4 +- tests/poly1305_vectors.rs | 10 +- tests/portable_fallback.rs | 4 +- tests/rapidhash_differential.rs | 4 +- tests/root_surface.rs | 469 +++-- tests/rsa_allocations.rs | 162 +- tests/rsa_leakage.rs | 60 +- tests/rsa_nist_cavp.rs | 139 +- tests/rsa_profile_confusion.rs | 36 +- tests/rsa_public_key.rs | 1572 ++++++++------- tests/rsa_wycheproof.rs | 140 +- tests/scrypt_differential.rs | 32 +- tests/secret_redaction.rs | 30 +- tests/serde_roundtrip.rs | 20 +- tests/sha256_differential.rs | 8 +- tests/sha256_official_vectors.rs | 6 +- tests/sha2_official_vectors.rs | 6 +- tests/sha3_differential.rs | 16 +- tests/sha3_official_vectors.rs | 10 +- tests/sha512_differential.rs | 12 +- tests/shake128_differential.rs | 6 +- tests/shake256_differential.rs | 6 +- tests/support/blobby_compat.rs | 86 +- tests/support/mod.rs | 2 +- tests/x25519_vectors.rs | 64 +- tests/x25519_wycheproof.rs | 36 +- tests/xchacha20poly1305.rs | 70 +- tools/ct-binsec-harness/src/main.rs | 131 +- tools/ct-dudect/Cargo.lock | 8 +- tools/ct-dudect/src/main.rs | 529 +++-- tools/ct-harness/src/lib.rs | 1006 ++++++++-- tools/ct-harness/src/main.rs | 2 +- tools/wasm-runtime-vectors/Cargo.lock | 2 +- tools/wasm-runtime-vectors/src/main.rs | 20 +- 634 files changed, 28394 insertions(+), 25565 deletions(-) create mode 100644 .changes/add-debug-output-for-4cec.md create mode 100644 .changes/add-debug-output-for-6a2e.md create mode 100644 .changes/add-redacted-debug-output-4c0e.md create mode 100644 .changes/document-checksum-and-digest-e536.md create mode 100644 .changes/document-shake128-and-shake256-2928.md create mode 100644 .changes/harden-aegis256-aesni-a37b.md create mode 100644 .changes/harden-blake2b-length-handling-4ab7.md create mode 100644 .changes/harden-blake2s-length-handling-454e.md create mode 100644 .changes/harden-blake3-tree-arithmetic-32c1.md create mode 100644 .changes/harden-ecdsa-limb-boundaries-6503.md create mode 100644 .changes/harden-ed25519-simd-boundaries-de52.md create mode 100644 .changes/harden-ml-kem-encoding-2483.md create mode 100644 .changes/harden-pbkdf2-block-indexing-54fe.md create mode 100644 .changes/remove-unused-riscv-crc64-surfac-b7f9.md create mode 100644 .changes/report-unavailable-argon2-backe-8b41.md create mode 100644 .changes/unsafe-chacha20-diagnostic-backe-51d5.md create mode 100644 .config/toolchains.toml create mode 100644 .zed/settings.json create mode 100644 .zed/tasks.json create mode 100644 clippy.toml create mode 100755 scripts/check/lint-independent-workspaces.sh create mode 100755 scripts/ci/check-locked-cargo-test.sh create mode 100755 scripts/ci/check-locked-cargo.sh delete mode 100644 src/aead/aegis256/riscv64_vperm.rs delete mode 100644 src/aead/aes/riscv64_vperm_aes.rs delete mode 100644 src/checksum/crc64/riscv64.rs create mode 100644 tests/common/array.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 9768afc5..ddbe09f3 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,67 +1,2 @@ -# Linux x86_64 -[target.x86_64-unknown-linux-gnu] -rustflags = ["-C", "target-cpu=native"] - -# Linux ARM64 -[target.aarch64-unknown-linux-gnu] -rustflags = ["-C", "target-feature=+lse", "-C", "target-cpu=native"] - -# Windows x86_64 -[target.x86_64-pc-windows-msvc] - -# Windows ARM64 -[target.aarch64-pc-windows-msvc] -rustflags = ["-C", "target-feature=+lse"] - -# Tier B -# Linux x86_64 MUSL -[target.x86_64-unknown-linux-musl] - -# Linux ARM64 MUSL -[target.aarch64-unknown-linux-musl] -rustflags = ["-C", "target-feature=+lse"] - -# Tier C - WASM -# WASM - Browser -[target.wasm32-unknown-unknown] - -# WASM/WASI - Server -[target.wasm32-wasip1] - -# Tier D - Enterprise -# IBM s390x (Z mainframes) -[target.s390x-unknown-linux-gnu] - -# IBM PowerPC64 (little-endian, modern POWER systems) -[target.powerpc64le-unknown-linux-gnu] - -# RISC-V 64-bit Linux (RV64GC: general + compressed) -[target.riscv64gc-unknown-linux-gnu] - -# LoongArch 64-bit Linux -[target.loongarch64-unknown-linux-gnu] - -# Tier E - no_std Bare Metal Sentinels -# Cortex-M0/M0+ (minimal no_std sentinel - no FPU, no DSP) -# If it compiles here, it compiles everywhere -[target.thumbv6m-none-eabi] - -# ARM64 bare metal (modern no_std sentinel - hypervisors, TEEs) -# No LSE: target hardware capabilities vary -[target.aarch64-unknown-none] - -# RISC-V 64-bit bare metal (RV64IMAFDC) -[target.riscv64gc-unknown-none-elf] - -# RISC-V 32-bit bare metal (RV32IMAC) -[target.riscv32imac-unknown-none-elf] - -# x86_64 bare metal (kernels, bootloaders, hypervisors) -[target.x86_64-unknown-none] - -# Build Configuration [build] rustdocflags = ["-D", "warnings"] - -[env] -RUST_LOG = "info" diff --git a/.changes/add-debug-output-for-4cec.md b/.changes/add-debug-output-for-4cec.md new file mode 100644 index 00000000..7f386f2c --- /dev/null +++ b/.changes/add-debug-output-for-4cec.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Add Debug output for Ed25519 verification diagnostic snapshots. diff --git a/.changes/add-debug-output-for-6a2e.md b/.changes/add-debug-output-for-6a2e.md new file mode 100644 index 00000000..52f38010 --- /dev/null +++ b/.changes/add-debug-output-for-6a2e.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Add Debug output for AES-GCM nonce-counter state. diff --git a/.changes/add-redacted-debug-output-4c0e.md b/.changes/add-redacted-debug-output-4c0e.md new file mode 100644 index 00000000..5a8a312e --- /dev/null +++ b/.changes/add-redacted-debug-output-4c0e.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "major" +--- + +Group caller-supplied RSA blinding inputs in `RsaBlindingPair` and add redacted `Debug` output for borrowed RSA private-key inputs. diff --git a/.changes/document-checksum-and-digest-e536.md b/.changes/document-checksum-and-digest-e536.md new file mode 100644 index 00000000..35942df7 --- /dev/null +++ b/.changes/document-checksum-and-digest-e536.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Document checksum and digest I/O adapter state and ownership methods. diff --git a/.changes/document-shake128-and-shake256-2928.md b/.changes/document-shake128-and-shake256-2928.md new file mode 100644 index 00000000..a6c482da --- /dev/null +++ b/.changes/document-shake128-and-shake256-2928.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Document SHAKE128 and SHAKE256 state and output-reader APIs. diff --git a/.changes/harden-aegis256-aesni-a37b.md b/.changes/harden-aegis256-aesni-a37b.md new file mode 100644 index 00000000..46d4b9b3 --- /dev/null +++ b/.changes/harden-aegis256-aesni-a37b.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden the x86 and POWER8 AEGIS-256 backends against out-of-bounds pointer formation while preserving oracle-compatible ciphertext and tags. diff --git a/.changes/harden-blake2b-length-handling-4ab7.md b/.changes/harden-blake2b-length-handling-4ab7.md new file mode 100644 index 00000000..3c2dfee8 --- /dev/null +++ b/.changes/harden-blake2b-length-handling-4ab7.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden Blake2b length handling, state cleanup, and portable diagnostics. diff --git a/.changes/harden-blake2s-length-handling-454e.md b/.changes/harden-blake2s-length-handling-454e.md new file mode 100644 index 00000000..694dba0a --- /dev/null +++ b/.changes/harden-blake2s-length-handling-454e.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden Blake2s length handling, state cleanup, and portable diagnostics. diff --git a/.changes/harden-blake3-tree-arithmetic-32c1.md b/.changes/harden-blake3-tree-arithmetic-32c1.md new file mode 100644 index 00000000..193fb694 --- /dev/null +++ b/.changes/harden-blake3-tree-arithmetic-32c1.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden BLAKE3 tree and SIMD kernel arithmetic, bounds, and assembly ABI validation. diff --git a/.changes/harden-ecdsa-limb-boundaries-6503.md b/.changes/harden-ecdsa-limb-boundaries-6503.md new file mode 100644 index 00000000..6dcdd27f --- /dev/null +++ b/.changes/harden-ecdsa-limb-boundaries-6503.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden ECDSA limb arithmetic, endian conversion, diagnostic documentation, and fixed-work RISC-V and s390x multiplication while preserving P-256 and P-384 signature semantics. diff --git a/.changes/harden-ed25519-simd-boundaries-de52.md b/.changes/harden-ed25519-simd-boundaries-de52.md new file mode 100644 index 00000000..221f8db8 --- /dev/null +++ b/.changes/harden-ed25519-simd-boundaries-de52.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden Ed25519 scalar encoding, fixed-base table validation, table selection, and AVX2/AVX-512 IFMA safety boundaries while preserving signature and verification results. diff --git a/.changes/harden-ml-kem-encoding-2483.md b/.changes/harden-ml-kem-encoding-2483.md new file mode 100644 index 00000000..4c082bbe --- /dev/null +++ b/.changes/harden-ml-kem-encoding-2483.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden ML-KEM encoding, arithmetic bounds, and key-layout validation. diff --git a/.changes/harden-pbkdf2-block-indexing-54fe.md b/.changes/harden-pbkdf2-block-indexing-54fe.md new file mode 100644 index 00000000..2074cb96 --- /dev/null +++ b/.changes/harden-pbkdf2-block-indexing-54fe.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden PBKDF2 block indexing and document portable verification diagnostics. diff --git a/.changes/remove-unused-riscv-crc64-surfac-b7f9.md b/.changes/remove-unused-riscv-crc64-surfac-b7f9.md new file mode 100644 index 00000000..b18285c7 --- /dev/null +++ b/.changes/remove-unused-riscv-crc64-surfac-b7f9.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "major" +--- + +Remove the unused RISC-V CRC-64 Zbc/Zvbc force modes and dead accelerated backend; RISC-V CRC-64 continues to use the portable slice-by-16 implementation until target evidence justifies acceleration. diff --git a/.changes/report-unavailable-argon2-backe-8b41.md b/.changes/report-unavailable-argon2-backe-8b41.md new file mode 100644 index 00000000..7d02e777 --- /dev/null +++ b/.changes/report-unavailable-argon2-backe-8b41.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Return `Argon2Error::BackendUnavailable` when a forced diagnostic backend is unsupported. diff --git a/.changes/unsafe-chacha20-diagnostic-backe-51d5.md b/.changes/unsafe-chacha20-diagnostic-backe-51d5.md new file mode 100644 index 00000000..2c8fa02f --- /dev/null +++ b/.changes/unsafe-chacha20-diagnostic-backe-51d5.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "major" +--- + +Make the forced hardware ChaCha20 diagnostic functions `unsafe`; callers must establish the documented CPU capabilities before invoking them. diff --git a/.config/ci-tool-archives.tsv b/.config/ci-tool-archives.tsv index 75987b9d..fcda12a9 100644 --- a/.config/ci-tool-archives.tsv +++ b/.config/ci-tool-archives.tsv @@ -1,6 +1,6 @@ # tool version os architecture filename url sha256 -wasmtime v46.0.1 linux x86_64 wasmtime-v46.0.1-x86_64-linux.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v46.0.1/wasmtime-v46.0.1-x86_64-linux.tar.xz 9ae0b17ea298bcc52277a8208d6ab7fae8e1a89579672f9d82f9d86c116edb62 -wasmtime v46.0.1 linux aarch64 wasmtime-v46.0.1-aarch64-linux.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v46.0.1/wasmtime-v46.0.1-aarch64-linux.tar.xz 071c4def2a08f0ebc95c52dfd4f2886eb697ba495804217cf76e13b09d70a1be -wasmtime v46.0.1 macos x86_64 wasmtime-v46.0.1-x86_64-macos.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v46.0.1/wasmtime-v46.0.1-x86_64-macos.tar.xz 0513db67e7089c7e5f743a01427782bc4def83854222f4bc9b1d75f0b925240b -wasmtime v46.0.1 macos aarch64 wasmtime-v46.0.1-aarch64-macos.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v46.0.1/wasmtime-v46.0.1-aarch64-macos.tar.xz acee50be70dbe90b0ab2ac7db1321fc44715153a1b1cc58291c97b6d7cffc558 +wasmtime v47.0.3 linux x86_64 wasmtime-v47.0.3-x86_64-linux.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v47.0.3/wasmtime-v47.0.3-x86_64-linux.tar.xz ca1fc56d1afc40c8782e96c297fd182a0da162f9a8f52a1e7b094e1dd648e178 +wasmtime v47.0.3 linux aarch64 wasmtime-v47.0.3-aarch64-linux.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v47.0.3/wasmtime-v47.0.3-aarch64-linux.tar.xz 497b518db00ae585f04390758eaa99ad555bee50612dce7d102602778fb46ff0 +wasmtime v47.0.3 macos x86_64 wasmtime-v47.0.3-x86_64-macos.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v47.0.3/wasmtime-v47.0.3-x86_64-macos.tar.xz 424a50f76a9dcf4d02dab326b2374be1ad404030576ee915866e4af106058b35 +wasmtime v47.0.3 macos aarch64 wasmtime-v47.0.3-aarch64-macos.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v47.0.3/wasmtime-v47.0.3-aarch64-macos.tar.xz c2684249e5d9ef9351942cf2d315982cf201fe0300f05d63bc1527446f0cd37f codecov v11.3.1 linux x86_64 codecov https://cli.codecov.io/v11.3.1/linux/codecov ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d diff --git a/.config/nextest.toml b/.config/nextest.toml index 2635257e..91ab73c0 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -15,6 +15,10 @@ slow-timeout = { period = "60s", terminate-after = 8 } filter = 'binary(=rsa_wycheproof) & test(=wycheproof_rsaes_pkcs1v15_3072_decrypt_vectors_match_expected_results)' slow-timeout = { period = "60s", terminate-after = 8 } +[[profile.default.overrides]] +filter = 'binary(=rsa_wycheproof) & (test(=wycheproof_oaep_mgf1sha1_vectors_are_rejected_by_sha2_mgf1sha2_profiles) | test(=wycheproof_pkcs1v15_sha2_sig_gen_vectors_match_expected_signatures))' +slow-timeout = { period = "60s", terminate-after = 4 } + [profile.commit] status-level = "fail" success-output = "never" diff --git a/.config/rail.toml b/.config/rail.toml index 41f90f4e..ef09c902 100644 --- a/.config/rail.toml +++ b/.config/rail.toml @@ -4,23 +4,21 @@ # Targets targets = [ "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", "aarch64-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu", "aarch64-unknown-linux-musl", - "wasm32-unknown-unknown", - "wasm32-wasip1", - "x86_64-unknown-linux-musl", - "aarch64-pc-windows-msvc", "aarch64-unknown-none", - "loongarch64-unknown-linux-gnu", "powerpc64le-unknown-linux-gnu", "riscv32imac-unknown-none-elf", "riscv64gc-unknown-linux-gnu", - "riscv64gc-unknown-none-elf", "s390x-unknown-linux-gnu", "thumbv6m-none-eabi", + "wasm32-unknown-unknown", + "wasm32-wasip1", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", "x86_64-unknown-none", ] diff --git a/.config/target-matrix.json b/.config/target-matrix.json index 938a59ca..9294bc28 100644 --- a/.config/target-matrix.json +++ b/.config/target-matrix.json @@ -1,28 +1,28 @@ { "groups": { "win": [ - "x86_64-pc-windows-msvc", - "aarch64-pc-windows-msvc" + "aarch64-pc-windows-msvc", + "x86_64-pc-windows-msvc" ], "macos": [ "aarch64-apple-darwin", "x86_64-apple-darwin" ], "linux": [ - "x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", "riscv64gc-unknown-linux-gnu", - "x86_64-unknown-linux-musl", - "aarch64-unknown-linux-musl" + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl" ], "ibm": [ - "s390x-unknown-linux-gnu", - "powerpc64le-unknown-linux-gnu" + "powerpc64le-unknown-linux-gnu", + "s390x-unknown-linux-gnu" ], "no_std": [ - "thumbv6m-none-eabi", - "riscv32imac-unknown-none-elf", "aarch64-unknown-none", + "riscv32imac-unknown-none-elf", + "thumbv6m-none-eabi", "x86_64-unknown-none" ], "wasm": [ @@ -31,9 +31,9 @@ ] }, "ci": [ - { "name": "x86_64-unknown-linux-gnu", "type": "runson", "pool": "linux-x64-ci" }, + { "name": "aarch64-pc-windows-msvc", "type": "gha", "runner": "windows-11-arm" }, { "name": "aarch64-unknown-linux-gnu", "type": "runson", "pool": "linux-arm64-ci" }, { "name": "x86_64-pc-windows-msvc", "type": "gha", "runner": "windows-latest" }, - { "name": "aarch64-pc-windows-msvc", "type": "gha", "runner": "windows-11-arm" } + { "name": "x86_64-unknown-linux-gnu", "type": "runson", "pool": "linux-x64-ci" } ] } diff --git a/.config/toolchains.toml b/.config/toolchains.toml new file mode 100644 index 00000000..034d6d27 --- /dev/null +++ b/.config/toolchains.toml @@ -0,0 +1 @@ +nightly = "nightly-2026-08-12" diff --git a/.github/actions/setup-toolchain/action.yaml b/.github/actions/setup-toolchain/action.yaml index 9f3f968c..caf2ee10 100644 --- a/.github/actions/setup-toolchain/action.yaml +++ b/.github/actions/setup-toolchain/action.yaml @@ -1,9 +1,12 @@ -name: Setup Rust Toolchain (Dynamic) +name: Setup Rust Toolchain description: | - Install Rust toolchain reading version from rust-toolchain.toml. - Single source of truth for toolchain version. + Install the stable development toolchain, MSRV, or pinned nightly contract. inputs: + contract: + description: Toolchain contract to install (development, msrv, or nightly) + required: false + default: "development" components: description: Comma-separated list of components to install required: false @@ -11,7 +14,7 @@ inputs: outputs: toolchain: - description: The toolchain version that was installed + description: Exact installed toolchain version value: ${{ steps.read-toolchain.outputs.toolchain }} runs: @@ -20,8 +23,15 @@ runs: - name: Read Toolchain Version id: read-toolchain shell: bash + env: + TOOLCHAIN_CONTRACT: ${{ inputs.contract }} run: | - TOOLCHAIN=$(bash scripts/lib/toolchain.sh) + case "$TOOLCHAIN_CONTRACT" in + development) TOOLCHAIN=$(bash scripts/lib/toolchain.sh) ;; + msrv) TOOLCHAIN=$(bash scripts/lib/toolchain.sh --msrv) ;; + nightly) TOOLCHAIN=$(bash scripts/lib/toolchain.sh --nightly) ;; + *) echo "Unsupported toolchain contract: $TOOLCHAIN_CONTRACT" >&2; exit 2 ;; + esac echo "toolchain=$TOOLCHAIN" >> "$GITHUB_OUTPUT" echo "Using toolchain: $TOOLCHAIN" diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index a9f05709..99a2b35d 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -4,6 +4,10 @@ description: | Unified host setup for GHA + runs-on.com + IBM/RISC-V runners. inputs: + toolchain-contract: + description: Toolchain contract to install (development, msrv, or nightly). + required: false + default: "development" cache-key: description: Suffix for the rust-cache key (workflow + target). required: false @@ -31,11 +35,12 @@ runs: # runs-on.com S3-backed cache intercept. No-op on non-runs-on pools. - name: Setup MagicCache if: inputs.enable-magic-cache == 'true' - uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0 + uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0 - name: Setup Toolchain uses: ./.github/actions/setup-toolchain with: + contract: ${{ inputs.toolchain-contract }} components: ${{ inputs.toolchain-components }} - name: Install Cargo Tools @@ -47,7 +52,7 @@ runs: - name: Setup Rust Cache if: inputs.enable-rust-cache == 'true' - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: shared-key: "rscrypto-v1-${{ inputs.cache-key }}" cache-directories: ${{ inputs.tools-mode == 'rail' && 'target/cargo-rail/cache' || '' }} diff --git a/.github/workflows/_ci-suite.yaml b/.github/workflows/_ci-suite.yaml index aecc3dd7..f4c7357c 100644 --- a/.github/workflows/_ci-suite.yaml +++ b/.github/workflows/_ci-suite.yaml @@ -67,6 +67,20 @@ jobs: enable_rust_cache: true operation: quality + msrv: + name: MSRV + uses: ./.github/workflows/_rust-job.yaml + with: + runner: runs-on=${{ github.run_id }}/runner=linux-x64-ci + timeout_minutes: 30 + cache_key: ${{ inputs.cache_key_prefix }}-msrv + tools_mode: none + toolchain_contract: msrv + toolchain_components: "" + enable_magic_cache: true + enable_rust_cache: true + operation: msrv + cargo-graph: if: ${{ inputs.include_cargo_graph }} name: Cargo Graph Assurance @@ -126,6 +140,7 @@ jobs: timeout_minutes: 120 cache_key: ${{ inputs.cache_key_prefix }}-ibm-s390x tools_mode: none + toolchain_contract: nightly toolchain_components: "clippy, rustfmt" enable_magic_cache: false enable_rust_cache: true @@ -140,6 +155,7 @@ jobs: timeout_minutes: 120 cache_key: ${{ inputs.cache_key_prefix }}-ibm-power10 tools_mode: none + toolchain_contract: nightly toolchain_components: "clippy, rustfmt" enable_magic_cache: false enable_rust_cache: true @@ -154,6 +170,7 @@ jobs: timeout_minutes: 30 cache_key: ${{ inputs.cache_key_prefix }}-platform-amx-test-nodebug tools_mode: none + toolchain_contract: nightly toolchain_components: "rustfmt" enable_magic_cache: true enable_rust_cache: true @@ -167,6 +184,7 @@ jobs: timeout_minutes: 90 cache_key: ${{ inputs.cache_key_prefix }}-cross-targets tools_mode: none + toolchain_contract: nightly toolchain_components: "clippy, rust-src" enable_magic_cache: true enable_rust_cache: true diff --git a/.github/workflows/_rust-job.yaml b/.github/workflows/_rust-job.yaml index 8508c2de..e0ef686c 100644 --- a/.github/workflows/_rust-job.yaml +++ b/.github/workflows/_rust-job.yaml @@ -25,6 +25,10 @@ on: required: false type: string default: standard + toolchain_contract: + required: false + type: string + default: development toolchain_components: required: false type: string @@ -128,6 +132,7 @@ jobs: - name: Setup uses: ./.github/actions/setup with: + toolchain-contract: ${{ inputs.toolchain_contract }} cache-key: ${{ inputs.cache_key }} tools-mode: ${{ inputs.tools_mode }} toolchain-components: ${{ inputs.toolchain_components }} diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 26417e10..a3ebd8fe 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -71,6 +71,7 @@ jobs: timeout_minutes: ${{ matrix.timeout_minutes }} cache_key: bench-${{ matrix.artifact_suffix }} tools_mode: ${{ matrix.tools_mode }} + toolchain_contract: ${{ (matrix.platform == 'ibm-s390x' || matrix.platform == 'ibm-power10') && 'nightly' || 'development' }} toolchain_components: ${{ matrix.toolchain_components }} enable_magic_cache: false enable_rust_cache: true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6d00ab7f..6e890908 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -39,7 +39,7 @@ jobs: id: rail uses: loadingalias/cargo-rail-action@ebc8ef2740efefe2cd5c42826344a8f1213113ae # v6.1.2 with: - version: "0.20.0" + version: "0.21.0" checksum: required since: ${{ github.event.pull_request.base.sha }} diff --git a/.github/workflows/ct.yaml b/.github/workflows/ct.yaml index 569c55ea..0c7fabbf 100644 --- a/.github/workflows/ct.yaml +++ b/.github/workflows/ct.yaml @@ -175,6 +175,7 @@ jobs: timeout_minutes: ${{ matrix.timeout_minutes }} cache_key: ct-${{ matrix.artifact_suffix }} tools_mode: ${{ matrix.tools_mode }} + toolchain_contract: ${{ (matrix.platform == 'ibm-s390x' || matrix.platform == 'ibm-power10' || matrix.platform == 'rise-riscv') && 'nightly' || 'development' }} toolchain_components: ${{ matrix.toolchain_components }} enable_magic_cache: ${{ matrix.enable_magic_cache }} enable_rust_cache: ${{ matrix.enable_rust_cache }} diff --git a/.github/workflows/riscv.yaml b/.github/workflows/riscv.yaml index f6ac5a5d..3f8ae671 100644 --- a/.github/workflows/riscv.yaml +++ b/.github/workflows/riscv.yaml @@ -52,6 +52,7 @@ jobs: timeout_minutes: 240 cache_key: weekly-riscv64 tools_mode: none + toolchain_contract: nightly toolchain_components: "clippy, rustfmt" enable_magic_cache: false enable_rust_cache: true @@ -83,6 +84,7 @@ jobs: timeout_minutes: 240 cache_key: bench-rise-riscv tools_mode: ibm + toolchain_contract: nightly toolchain_components: "clippy, rustfmt" enable_magic_cache: false enable_rust_cache: true diff --git a/.github/workflows/rsa.yaml b/.github/workflows/rsa.yaml index 4bf3fa1b..b3b0d714 100644 --- a/.github/workflows/rsa.yaml +++ b/.github/workflows/rsa.yaml @@ -38,6 +38,7 @@ jobs: timeout_minutes: 60 cache_key: rsa-miri-linux-x64 tools_mode: minimal + toolchain_contract: nightly toolchain_components: "miri, rust-src" enable_magic_cache: true enable_rust_cache: true diff --git a/.github/workflows/weekly.yaml b/.github/workflows/weekly.yaml index 29b95b1b..cb9de23e 100644 --- a/.github/workflows/weekly.yaml +++ b/.github/workflows/weekly.yaml @@ -90,6 +90,7 @@ jobs: timeout_minutes: 60 cache_key: weekly-miri-${{ matrix.target.name }} tools_mode: minimal + toolchain_contract: nightly toolchain_components: "miri, rust-src" enable_magic_cache: true enable_rust_cache: true @@ -104,6 +105,7 @@ jobs: timeout_minutes: 60 cache_key: weekly-miri-tree-borrows tools_mode: minimal + toolchain_contract: nightly toolchain_components: "miri, rust-src" enable_magic_cache: true enable_rust_cache: true @@ -128,6 +130,7 @@ jobs: timeout_minutes: 120 cache_key: weekly-fuzz-${{ matrix.target.name }} tools_mode: fuzz + toolchain_contract: nightly enable_magic_cache: true enable_rust_cache: true operation: fuzz @@ -155,6 +158,7 @@ jobs: timeout_minutes: 120 cache_key: weekly-asan-fuzz-${{ matrix.target.name }} tools_mode: fuzz + toolchain_contract: nightly toolchain_components: "rust-src" enable_magic_cache: true enable_rust_cache: true diff --git a/.gitignore b/.gitignore index d9aa2a11..cac10004 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,9 @@ Thumbs.db AGENTS.md # Zed/IDE -.zed +.zed/* +!.zed/settings.json +!.zed/tasks.json # Keys/Creds *.pem diff --git a/.zed/settings.json b/.zed/settings.json new file mode 100644 index 00000000..e68e209b --- /dev/null +++ b/.zed/settings.json @@ -0,0 +1,13 @@ +{ + "lsp": { + "rust-analyzer": { + "enable_lsp_tasks": false, + "initialization_options": { + "cargo": { + "extraArgs": ["--locked"], + "features": "all" + } + } + } + } +} diff --git a/.zed/tasks.json b/.zed/tasks.json new file mode 100644 index 00000000..048cc3df --- /dev/null +++ b/.zed/tasks.json @@ -0,0 +1,18 @@ +[ + { + "label": "rscrypto: check", + "command": "just", + "args": ["check"], + "cwd": "$ZED_WORKTREE_ROOT", + "allow_concurrent_runs": false, + "save": "all" + }, + { + "label": "rscrypto: test", + "command": "just", + "args": ["test"], + "cwd": "$ZED_WORKTREE_ROOT", + "allow_concurrent_runs": false, + "save": "all" + } +] diff --git a/Cargo.lock b/Cargo.lock index 133c9ad3..46569232 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,16 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "aead" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" -dependencies = [ - "crypto-common 0.1.7", - "generic-array", -] - [[package]] name = "aead" version = "0.6.1" @@ -19,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ "crypto-common 0.2.2", - "inout 0.2.2", + "inout", ] [[package]] @@ -32,24 +22,13 @@ dependencies = [ "softaes", ] -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher 0.4.4", - "cpufeatures 0.2.17", -] - [[package]] name = "aes" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" dependencies = [ - "cipher 0.5.2", + "cipher", "cpubits", "cpufeatures 0.3.0", ] @@ -60,34 +39,33 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ - "aead 0.6.1", - "aes 0.9.2", - "cipher 0.5.2", - "ctr 0.10.1", + "aead", + "aes", + "cipher", + "ctr", "ghash", "subtle", ] [[package]] name = "aes-gcm-siv" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0784134ba9375416d469ec31e7c5f9fa94405049cf08c5ce5b4698be673e0d" +checksum = "1f437e5b075722bda3f54039e95d60d1c142f140e6c45b718e4f9fca3e5a1514" dependencies = [ - "aead 0.5.2", - "aes 0.8.4", - "cipher 0.4.4", - "ctr 0.9.2", - "polyval 0.6.2", + "aead", + "aes", + "cipher", + "ctr", + "polyval", "subtle", - "zeroize", ] [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -115,13 +93,13 @@ checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "argon2" -version = "0.6.0-rc.8" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7af50940b73bf4e16c15c448a2b121c63f2d68e3e54b6a8731673cb4aa0cdff5" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", "blake2", - "cpufeatures 0.3.0", + "cpufeatures 0.2.17", "password-hash", ] @@ -143,9 +121,9 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6cba922132fed390c20be89a16fa713999c46b2ce84ac4b0ecee0f3586a39d88" dependencies = [ - "aead 0.6.1", + "aead", "ascon-core", - "inout 0.2.2", + "inout", "subtle", ] @@ -173,32 +151,20 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.3" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" dependencies = [ - "aws-lc-sys 0.43.0", + "aws-lc-sys", "untrusted 0.7.1", "zeroize", ] [[package]] name = "aws-lc-sys" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", -] - -[[package]] -name = "aws-lc-sys" -version = "0.43.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" dependencies = [ "cc", "cmake", @@ -207,12 +173,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - [[package]] name = "base16ct" version = "1.0.0" @@ -268,18 +228,18 @@ checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "blake2" -version = "0.11.0-rc.6" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061f1a09225e328e1ffbb378d2d49923c0ca5fee19fb5ac1cc9c1e9d52b93690" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest 0.11.3", + "digest 0.10.7", ] [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" dependencies = [ "arrayref", "arrayvec", @@ -329,9 +289,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -361,7 +321,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cipher 0.5.2", + "cipher", "cpufeatures 0.3.0", "rand_core 0.10.1", "zeroize", @@ -373,9 +333,9 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b89e1c441e926b9c82a8d023f6e1b7ae0adcfaa7d621814e4d60789bac751cb" dependencies = [ - "aead 0.6.1", + "aead", "chacha20", - "cipher 0.5.2", + "cipher", "poly1305", ] @@ -406,16 +366,6 @@ dependencies = [ "half", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common 0.1.7", - "inout 0.1.4", -] - [[package]] name = "cipher" version = "0.5.2" @@ -424,7 +374,7 @@ checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ "block-buffer 0.12.1", "crypto-common 0.2.2", - "inout 0.2.2", + "inout", "zeroize", ] @@ -441,18 +391,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstyle", "clap_lex", @@ -657,18 +607,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" -[[package]] -name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - [[package]] name = "crypto-bigint" version = "0.7.5" @@ -692,7 +630,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", - "rand_core 0.6.4", "typenum", ] @@ -714,26 +651,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6250a2d96a09edbe8e75ed29c87d05512ee2cbb24c7e8c684657f7930ffd3c6" dependencies = [ "digest 0.11.3", - "keccak 0.2.0", + "keccak 0.2.1", "sponge-cursor", ] -[[package]] -name = "ctr" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" -dependencies = [ - "cipher 0.4.4", -] - [[package]] name = "ctr" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" dependencies = [ - "cipher 0.5.2", + "cipher", ] [[package]] @@ -755,8 +683,23 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "curve25519-dalek-derive", - "digest 0.10.7", - "fiat-crypto", + "fiat-crypto 0.2.9", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "curve25519-dalek-derive", + "digest 0.11.3", + "fiat-crypto 0.3.0", "rustc_version", "subtle", "zeroize", @@ -816,22 +759,23 @@ dependencies = [ "const-oid 0.10.2", "crypto-common 0.2.2", "ctutils", + "zeroize", ] [[package]] name = "dryoc" -version = "0.8.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d09fac1c1cf30622ccd33105965abb8ef17dd1c3aae34a489350dd0b8c3fae4" +checksum = "2a9268c01fc6ec1f2a104ae9e40343c255fbcafea97a48f334237206c96838b2" dependencies = [ - "bitflags", "chacha20", - "curve25519-dalek", - "lazy_static", + "curve25519-dalek 4.1.3", + "getrandom 0.4.3", "libc", "rand 0.10.2", "salsa20", "sha2 0.11.0", + "sha3 0.12.0", "subtle", "winapi", "zeroize", @@ -843,20 +787,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" -[[package]] -name = "ecdsa" -version = "0.16.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der 0.7.10", - "digest 0.10.7", - "elliptic-curve 0.13.8", - "rfc6979 0.4.0", - "signature 2.2.0", - "spki 0.7.3", -] - [[package]] name = "ecdsa" version = "0.17.0" @@ -865,8 +795,8 @@ checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ "der 0.8.1", "digest 0.11.3", - "elliptic-curve 0.14.1", - "rfc6979 0.6.0", + "elliptic-curve", + "rfc6979", "signature 3.0.0", "spki 0.8.0", "zeroize", @@ -874,24 +804,22 @@ dependencies = [ [[package]] name = "ed25519" -version = "2.2.3" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" dependencies = [ - "pkcs8 0.10.2", - "signature 2.2.0", + "signature 3.0.0", ] [[package]] name = "ed25519-dalek" -version = "2.2.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" dependencies = [ - "curve25519-dalek", + "curve25519-dalek 5.0.0", "ed25519", - "serde", - "sha2 0.10.9", + "sha2 0.11.0", "subtle", "zeroize", ] @@ -902,41 +830,22 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct 0.2.0", - "crypto-bigint 0.5.5", - "digest 0.10.7", - "ff 0.13.1", - "generic-array", - "group 0.13.0", - "pkcs8 0.10.2", - "rand_core 0.6.4", - "sec1 0.7.3", - "subtle", - "zeroize", -] - [[package]] name = "elliptic-curve" version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" dependencies = [ - "base16ct 1.0.0", - "crypto-bigint 0.7.5", + "base16ct", + "crypto-bigint", "crypto-common 0.2.2", "digest 0.11.3", - "ff 0.14.0", - "group 0.14.0", + "ff", + "group", "hybrid-array", "pkcs8 0.11.0", "rand_core 0.10.1", - "sec1 0.8.1", + "sec1", "subtle", "zeroize", ] @@ -957,16 +866,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "ff" version = "0.14.0" @@ -983,11 +882,17 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "fips203" @@ -1015,21 +920,21 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-task", @@ -1045,7 +950,6 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", - "zeroize", ] [[package]] @@ -1078,9 +982,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -1089,7 +995,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ - "polyval 0.7.3", + "polyval", ] [[package]] @@ -1098,24 +1004,13 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff 0.13.1", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "group" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" dependencies = [ - "ff 0.14.0", + "ff", "rand_core 0.10.1", "subtle", ] @@ -1174,16 +1069,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" dependencies = [ - "hmac 0.13.0", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", + "hmac", ] [[package]] @@ -1207,15 +1093,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] - [[package]] name = "inout" version = "0.2.2" @@ -1252,9 +1129,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ "cfg-if", "futures-util", @@ -1272,9 +1149,9 @@ dependencies = [ [[package]] name = "keccak" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +checksum = "ffd9697dc4a9a62e2da93389f34400b77a28f0287711263cabb203b3ccb9c0e4" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -1472,9 +1349,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -1511,35 +1388,31 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - [[package]] name = "p256" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a" dependencies = [ - "ecdsa 0.17.0", - "elliptic-curve 0.14.1", + "ecdsa", + "elliptic-curve", "primefield", - "primeorder 0.14.0", + "primeorder", "sha2 0.11.0", ] [[package]] name = "p384" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +checksum = "d17b851e6b3e378ab4ecb07fa2ed23f4d15f075735f8fec9fa1e7bdce5f8301f" dependencies = [ - "ecdsa 0.16.9", - "elliptic-curve 0.13.8", - "primeorder 0.13.6", - "sha2 0.10.9", + "ecdsa", + "elliptic-curve", + "fiat-crypto 0.3.0", + "primefield", + "primeorder", + "sha2 0.11.0", ] [[package]] @@ -1554,11 +1427,13 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.6.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ - "phc", + "base64ct", + "rand_core 0.6.4", + "subtle", ] [[package]] @@ -1574,17 +1449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ "digest 0.11.3", - "hmac 0.13.0", -] - -[[package]] -name = "phc" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" -dependencies = [ - "base64ct", - "ctutils", + "hmac", ] [[package]] @@ -1665,19 +1530,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2d0073b297041425c7c3df6eb4792d598a15323fe63346852b092eca02904c" dependencies = [ "cpufeatures 0.3.0", - "universal-hash 0.6.1", -] - -[[package]] -name = "polyval" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "opaque-debug", - "universal-hash 0.5.1", + "universal-hash", ] [[package]] @@ -1688,7 +1541,7 @@ checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ "cpubits", "cpufeatures 0.3.0", - "universal-hash 0.6.1", + "universal-hash", ] [[package]] @@ -1716,30 +1569,21 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" dependencies = [ - "crypto-bigint 0.7.5", + "crypto-bigint", "crypto-common 0.2.2", - "ff 0.14.0", + "ff", "rand_core 0.10.1", "subtle", "zeroize", ] -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve 0.13.8", -] - [[package]] name = "primeorder" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" dependencies = [ - "elliptic-curve 0.14.1", + "elliptic-curve", "once_cell", "primefield", "serdect", @@ -1950,9 +1794,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -1965,24 +1809,14 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac 0.12.1", - "subtle", -] - [[package]] name = "rfc6979" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" dependencies = [ - "crypto-bigint 0.7.5", - "hmac 0.13.0", + "crypto-bigint", + "hmac", ] [[package]] @@ -2031,7 +1865,7 @@ dependencies = [ "ascon-aead", "ascon-hash", "aws-lc-rs", - "aws-lc-sys 0.41.0", + "aws-lc-sys", "blake2", "blake3", "chacha20poly1305", @@ -2048,8 +1882,8 @@ dependencies = [ "fips203", "getrandom 0.4.3", "hkdf", - "hmac 0.13.0", - "keccak 0.2.0", + "hmac", + "keccak 0.2.1", "libcrux-ml-kem", "ml-kem", "p256", @@ -2124,7 +1958,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f874456e72520ff1375a06c588eaf074b0f01f9e9e1aada45bd9b7954a6e42c" dependencies = [ "cfg-if", - "cipher 0.5.2", + "cipher", ] [[package]] @@ -2148,27 +1982,13 @@ dependencies = [ "sha2 0.11.0", ] -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct 0.2.0", - "der 0.7.10", - "generic-array", - "pkcs8 0.10.2", - "subtle", - "zeroize", -] - [[package]] name = "sec1" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" dependencies = [ - "base16ct 1.0.0", + "base16ct", "ctutils", "der 0.8.1", "hybrid-array", @@ -2231,7 +2051,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" dependencies = [ - "base16ct 1.0.0", + "base16ct", "serde", ] @@ -2274,7 +2094,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" dependencies = [ "digest 0.11.3", - "keccak 0.2.0", + "keccak 0.2.1", ] [[package]] @@ -2284,7 +2104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc9bad02c26382724b2d2692c6f179285e4b54eeecd7968f52a50059c3c11759" dependencies = [ "digest 0.11.3", - "keccak 0.2.0", + "keccak 0.2.1", "sponge-cursor", ] @@ -2375,6 +2195,9 @@ name = "sponge-cursor" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" +dependencies = [ + "zeroize", +] [[package]] name = "subtle" @@ -2454,16 +2277,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "universal-hash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" -dependencies = [ - "crypto-common 0.1.7", - "subtle", -] - [[package]] name = "universal-hash" version = "0.6.1" @@ -2539,9 +2352,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -2552,9 +2365,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2562,9 +2375,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", @@ -2575,18 +2388,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" dependencies = [ "js-sys", "wasm-bindgen", @@ -2723,20 +2536,19 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1" dependencies = [ - "ff 0.14.0", - "group 0.14.0", + "ff", + "group", "hybrid-array", ] [[package]] name = "x25519-dalek" -version = "2.0.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +checksum = "e7e8131a03190127fb2263afc72b322ecadae46b6ff8c6f399ff5d02f5559af6" dependencies = [ - "curve25519-dalek", - "rand_core 0.6.4", - "serde", + "curve25519-dalek 5.0.0", + "rand_core 0.10.1", "zeroize", ] @@ -2748,18 +2560,18 @@ checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "zerocopy" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 2d718ff7..108993d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,36 +47,101 @@ rustdoc-args = ["--cfg", "docsrs"] resolver = "3" members = ["."] -[workspace.lints.rust] -unsafe_code = "allow" +[lints.rust] +warnings = { level = "deny", priority = -1 } +ambiguous_negative_literals = "deny" unsafe_op_in_unsafe_fn = "deny" -dangling_pointers_from_locals = "deny" -integer_to_ptr_transmutes = "deny" -unexpected_cfgs = { level = "warn", check-cfg = [ +ffi_unwind_calls = "deny" +let_underscore_drop = "deny" +unreachable_pub = "deny" +unexpected_cfgs = { level = "deny", check-cfg = [ 'cfg(miri)', 'cfg(fuzzing)', 'cfg(rscrypto_internal_fuzzing)', 'cfg(target_feature, values("movdiri", "movdir64b", "serialize"))', ] } -[workspace.lints.clippy] -correctness = { level = "deny", priority = -1 } -suspicious = { level = "deny", priority = -1 } -perf = { level = "deny", priority = -1 } +[lints.clippy] +all = { level = "deny", priority = -1 } + +non_send_fields_in_send_ty = "deny" +significant_drop_in_scrutinee = "deny" + +panic = "deny" +panic_in_result_fn = "deny" +string_slice = "deny" +todo = "deny" +unchecked_time_subtraction = "deny" +unimplemented = "deny" +unreachable = "deny" +unwrap_used = "deny" + +assertions_on_result_states = "deny" +let_underscore_must_use = "deny" +unused_result_ok = "deny" + +cast_possible_truncation = "deny" +cast_possible_wrap = "deny" +cast_sign_loss = "deny" +checked_conversions = "deny" +fn_to_numeric_cast_any = "deny" +invalid_upcast_comparisons = "deny" +lossy_float_literal = "deny" + +allow_attributes = "deny" +allow_attributes_without_reason = "deny" +dbg_macro = "deny" +enum_glob_use = "deny" +expl_impl_clone_on_copy = "deny" +iter_not_returning_iterator = "deny" +same_functions_in_if_condition = "deny" + +negative_feature_names = "deny" +wildcard_dependencies = "deny" + +coerce_container_to_any = "deny" +collection_is_never_read = "deny" +debug_assert_with_mut_call = "deny" +default_union_representation = "deny" +fallible_impl_from = "deny" +ignore_without_reason = "deny" +literal_string_with_formatting_args = "deny" +mixed_read_write_in_expression = "deny" +needless_collect = "deny" +or_fun_call = "deny" +path_buf_push_overwrite = "deny" +read_zero_byte_vec = "deny" +redundant_clone = "deny" +should_panic_without_expect = "deny" +suspicious_operation_groupings = "deny" +trait_duplication_in_bounds = "deny" +transmute_undefined_repr = "deny" +uninhabited_references = "deny" +unsafe_derive_deserialize = "deny" +volatile_composites = "deny" + +as_ptr_cast_mut = "deny" +as_pointer_underscore = "deny" +borrow_as_ptr = "deny" +cast_ptr_alignment = "deny" +mem_forget = "deny" +ptr_as_ptr = "deny" +ptr_cast_constness = "deny" +ref_as_ptr = "deny" +undocumented_unsafe_blocks = "deny" +unnecessary_safety_comment = "deny" +unnecessary_safety_doc = "deny" -std_instead_of_core = "warn" -std_instead_of_alloc = "warn" -unwrap_used = "allow" -expect_used = "allow" -indexing_slicing = "allow" +alloc_instead_of_core = "deny" +std_instead_of_alloc = "deny" +std_instead_of_core = "deny" -undocumented_unsafe_blocks = "deny" -missing_safety_doc = "deny" -chunks_exact_to_as_chunks = "allow" +arithmetic_side_effects = "deny" +cargo_common_metadata = "deny" +cfg_not_test = "deny" -[workspace.lints.rustdoc] -broken_intra_doc_links = "deny" -private_intra_doc_links = "warn" +[lints.rustdoc] +all = "deny" [features] default = ["std"] @@ -164,36 +229,36 @@ crc32c = "0.6.8" crc32fast = "1.5.0" crc64fast = "1.1.0" ascon-hash = "0.4.0" -argon2 = { version = "0.6.0-rc.8", default-features = false, features = ["alloc"] } -blake3 = { version = "1.8.5", default-features = false, features = ["std", "rayon"] } -blake2 = "0.11.0-rc.6" +argon2 = { version = "0.5.3", default-features = false, features = ["alloc"] } +blake3 = { version = "1.8.6", default-features = false, features = ["std", "rayon"] } +blake2 = "0.10.6" pbkdf2 = "0.13.0" scrypt = { version = "0.12.0", default-features = false } sha2 = "0.11.0" sha2_010 = { package = "sha2", version = "0.10.9" } sha3 = "0.12.0" -keccak = "0.2.0" +keccak = "0.2.1" xxhash-rust = { version = "0.8.18", features = ["xxh3"] } rapidhash = "=4.5.1" tiny-keccak = { version = "2.0.2", features = ["kmac", "shake"] } digest = { version = "0.11.3", features = ["dev"] } p256 = { version = "0.14.0", default-features = false, features = ["ecdsa", "pkcs8", "std"] } -p384 = { version = "0.13.1", default-features = false, features = ["ecdsa", "pkcs8", "std"] } -ed25519-dalek = "2.2.0" -x25519-dalek = { version = "2.0.1", features = ["static_secrets"] } +p384 = { version = "0.14.0", default-features = false, features = ["ecdsa", "pkcs8", "std"] } +ed25519-dalek = "3.0.0" +x25519-dalek = { version = "3.0.0", features = ["static_secrets"] } fips203 = { version = "0.4.3", default-features = false, features = ["ml-kem-512", "ml-kem-768", "ml-kem-1024"] } rustcrypto-ml-kem = { package = "ml-kem", version = "0.3.2", default-features = false, features = ["hazmat"] } hmac = "0.13.0" hkdf = "0.13.0" chacha20poly1305 = "0.11.0" aes-gcm = "0.11.0" -aes-gcm-siv = "0.11.1" +aes-gcm-siv = "0.12.0" aegis = "0.9.15" ascon-aead = { version = "0.6", default-features = false, features = ["alloc"] } serde_json = "1" ring = "0.17.14" rsa = { version = "0.9.10", default-features = false, features = ["std", "u64_digit", "sha2"] } -dryoc = "0.8.0" +dryoc = "1.0.0" [target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dev-dependencies] libcrux-ml-kem = { version = "0.0.10", default-features = false, features = [ @@ -219,8 +284,8 @@ libcrux-ml-kem = { version = "0.0.10", default-features = false, features = [ ] } [target.'cfg(all(any(unix, windows), not(target_arch = "wasm32"), not(any(target_arch = "s390x", target_arch = "powerpc64"))))'.dev-dependencies] -aws-lc-rs = "1.17.3" -aws-lc-sys = "0.41.0" +aws-lc-rs = "1.18.0" +aws-lc-sys = "0.44.0" [lib] bench = false @@ -334,9 +399,6 @@ name = "kmac_cshake" harness = false required-features = ["sha3", "kmac"] -[lints] -workspace = true - [profile.dev] opt-level = 0 debug = true diff --git a/README.md b/README.md index d7271097..730050ca 100644 --- a/README.md +++ b/README.md @@ -39,14 +39,14 @@ Minimal `no_std` SHA-2 build: ```toml [dependencies] -rscrypto = { version = "0.8.0", default-features = false, features = ["sha2"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["sha2"] } ``` Full primitive stack with OS randomness enabled: ```toml [dependencies] -rscrypto = { version = "0.8.0", features = ["full", "getrandom"] } +rscrypto = { version = "0.8.1", features = ["full", "getrandom"] } ``` Use `default-features = false` for `no_std` builds. Enable `getrandom` only for @@ -195,9 +195,10 @@ methodology, and known losses are in The minimum supported Rust version is **1.91.0**. -The pinned development nightly in -[`rust-toolchain.toml`](rust-toolchain.toml) is separate from the MSRV and is -used for Miri, fuzzing, and architecture-specific checks. +The pinned stable development toolchain in +[`rust-toolchain.toml`](rust-toolchain.toml) is separate from the MSRV. +Nightly-only Miri, fuzzing, and architecture checks use the dated exception in +[`toolchains.toml`](.config/toolchains.toml). ## License diff --git a/benches/aead.rs b/benches/aead.rs index 8b40ff6c..743c9dc0 100644 --- a/benches/aead.rs +++ b/benches/aead.rs @@ -9,7 +9,6 @@ mod common; use core::hint::black_box; use aes_gcm::aead::{AeadInOut as _, KeyInit as _}; -use aes_gcm_siv::aead::{AeadInPlace as _, KeyInit as _}; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use rscrypto::aead::expert::AeadWithNonce; @@ -78,7 +77,7 @@ fn xchacha20_poly1305_encrypt(c: &mut Criterion) { black_box(AAD), black_box(buf.as_mut_slice().into()), ) - .unwrap(), + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -100,13 +99,15 @@ fn xchacha20_poly1305_decrypt(c: &mut Criterion) { // Pre-encrypt with rscrypto to get valid ciphertext + tag. let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); // Pre-encrypt with RustCrypto to get its tag format. let mut ct_rc = data.clone(); let tag_rc = cipher_rc .encrypt_inout_detached(&nonce_rc, AAD, ct_rc.as_mut_slice().into()) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); let mut buf = ciphertext.clone(); @@ -120,7 +121,7 @@ fn xchacha20_poly1305_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -137,7 +138,7 @@ fn xchacha20_poly1305_decrypt(c: &mut Criterion) { black_box(buf_rc.as_mut_slice().into()), black_box(&tag_rc), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_rc); }) }); @@ -161,10 +162,12 @@ fn chacha20_poly1305_encrypt(c: &mut Criterion) { let nonce_rc = chacha20poly1305::Nonce::from(NONCE_12); aws_lc_bench! { let aws_key = - aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::CHACHA20_POLY1305, &KEY_32).unwrap()); + aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::CHACHA20_POLY1305, &KEY_32).expect("valid AEAD benchmark operation must succeed")); } - let ring_key = - ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(&ring_aead::CHACHA20_POLY1305, &KEY_32).unwrap()); + let ring_key = ring_aead::LessSafeKey::new( + ring_aead::UnboundKey::new(&ring_aead::CHACHA20_POLY1305, &KEY_32) + .expect("valid AEAD benchmark operation must succeed"), + ); let mut g = c.benchmark_group("chacha20-poly1305/encrypt"); for (len, data) in &inputs { @@ -174,7 +177,7 @@ fn chacha20_poly1305_encrypt(c: &mut Criterion) { let mut buf_owned = data.clone(); #[cfg(all(feature = "diag", target_arch = "x86_64", target_os = "linux"))] let mut buf_x86_asm = data.clone(); - let mut buf_combined: Vec = Vec::with_capacity(data.len() + 16); + let mut buf_combined: Vec = Vec::with_capacity(data.len().strict_add(16)); g.bench_with_input(BenchmarkId::new("rscrypto", len), data, |b, d| { b.iter(|| { @@ -224,7 +227,7 @@ fn chacha20_poly1305_encrypt(c: &mut Criterion) { black_box(AAD), black_box(buf.as_mut_slice().into()), ) - .unwrap(), + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -240,7 +243,7 @@ fn chacha20_poly1305_encrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_combined), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_combined); }) }); @@ -256,7 +259,7 @@ fn chacha20_poly1305_encrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), black_box(&mut buf_combined), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_combined); }) }); @@ -278,22 +281,26 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { let nonce_rc = chacha20poly1305::Nonce::from(NONCE_12); aws_lc_bench! { let aws_key = - aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::CHACHA20_POLY1305, &KEY_32).unwrap()); + aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::CHACHA20_POLY1305, &KEY_32).expect("valid AEAD benchmark operation must succeed")); } - let ring_key = - ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(&ring_aead::CHACHA20_POLY1305, &KEY_32).unwrap()); + let ring_key = ring_aead::LessSafeKey::new( + ring_aead::UnboundKey::new(&ring_aead::CHACHA20_POLY1305, &KEY_32) + .expect("valid AEAD benchmark operation must succeed"), + ); let mut g = c.benchmark_group("chacha20-poly1305/decrypt"); for (len, data) in &inputs { common::set_throughput(&mut g, *len); let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); let mut ct_rc = data.clone(); let tag_rc = cipher_rc .encrypt_inout_detached(&nonce_rc, AAD, ct_rc.as_mut_slice().into()) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); aws_lc_bench! { let mut ct_aws: Vec = data.clone(); @@ -303,7 +310,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), &mut ct_aws, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); } let mut ct_ring: Vec = data.clone(); @@ -313,7 +320,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), &mut ct_ring, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); let mut buf = ciphertext.clone(); #[cfg(feature = "diag")] @@ -331,7 +338,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -347,7 +354,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { black_box(&mut buf_owned), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_owned); }) }); @@ -365,7 +372,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { black_box(&tag_rs), ) .expect("x86 asm path must apply to benchmarked non-empty sizes") - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_x86_asm); }) }); @@ -383,7 +390,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { black_box(buf_rc.as_mut_slice().into()), black_box(&tag_rc), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_rc); }) }); @@ -400,7 +407,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_aws); }) }); @@ -417,7 +424,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), black_box(&mut buf_ring), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_ring); }) }); @@ -437,10 +444,10 @@ fn aes256_gcm_siv_encrypt(c: &mut Criterion) { let nonce_rs = rscrypto::aead::Nonce96::from_bytes(NONCE_12); let cipher_rs = rscrypto::Aes256GcmSiv::new(&rscrypto::Aes256GcmSivKey::from_bytes(KEY_32)); let cipher_rc = aes_gcm_siv::Aes256GcmSiv::new(&KEY_32.into()); - let nonce_rc = aes_gcm_siv::Nonce::from_slice(&NONCE_12); + let nonce_rc = aes_gcm_siv::Nonce::from(NONCE_12); aws_lc_bench! { let aws_key = - aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM_SIV, &KEY_32).unwrap()); + aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM_SIV, &KEY_32).expect("valid AEAD benchmark operation must succeed")); } let mut g = c.benchmark_group("aes-256-gcm-siv/encrypt"); @@ -460,8 +467,12 @@ fn aes256_gcm_siv_encrypt(c: &mut Criterion) { buf.copy_from_slice(d); black_box( cipher_rc - .encrypt_in_place_detached(black_box(nonce_rc), black_box(AAD), black_box(&mut buf)) - .unwrap(), + .encrypt_inout_detached( + black_box(&nonce_rc), + black_box(AAD), + black_box(buf.as_mut_slice()).into(), + ) + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -477,7 +488,7 @@ fn aes256_gcm_siv_encrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(tag.as_ref()); black_box(&buf_aws); }) @@ -497,10 +508,10 @@ fn aes256_gcm_siv_decrypt(c: &mut Criterion) { let nonce_rs = rscrypto::aead::Nonce96::from_bytes(NONCE_12); let cipher_rs = rscrypto::Aes256GcmSiv::new(&rscrypto::Aes256GcmSivKey::from_bytes(KEY_32)); let cipher_rc = aes_gcm_siv::Aes256GcmSiv::new(&KEY_32.into()); - let nonce_rc = aes_gcm_siv::Nonce::from_slice(&NONCE_12); + let nonce_rc = aes_gcm_siv::Nonce::from(NONCE_12); aws_lc_bench! { let aws_key = - aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM_SIV, &KEY_32).unwrap()); + aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM_SIV, &KEY_32).expect("valid AEAD benchmark operation must succeed")); } let mut g = c.benchmark_group("aes-256-gcm-siv/decrypt"); @@ -508,10 +519,14 @@ fn aes256_gcm_siv_decrypt(c: &mut Criterion) { common::set_throughput(&mut g, *len); let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); let mut ct_rc = data.clone(); - let tag_rc = cipher_rc.encrypt_in_place_detached(nonce_rc, AAD, &mut ct_rc).unwrap(); + let tag_rc = cipher_rc + .encrypt_inout_detached(&nonce_rc, AAD, ct_rc.as_mut_slice().into()) + .expect("valid AEAD benchmark operation must succeed"); aws_lc_bench! { let mut ct_aws = data.clone(); @@ -521,7 +536,7 @@ fn aes256_gcm_siv_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), &mut ct_aws, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); // AWS-LC exposes detached tags for seal, but its in-place open API takes ct||tag. ct_aws.extend_from_slice(tag_aws.as_ref()); } @@ -538,7 +553,7 @@ fn aes256_gcm_siv_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -549,13 +564,13 @@ fn aes256_gcm_siv_decrypt(c: &mut Criterion) { b.iter(|| { buf_rc.copy_from_slice(ct); cipher_rc - .decrypt_in_place_detached( - black_box(nonce_rc), + .decrypt_inout_detached( + black_box(&nonce_rc), black_box(AAD), - black_box(&mut buf_rc), + black_box(buf_rc.as_mut_slice()).into(), black_box(&tag_rc), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_rc); }) }); @@ -572,7 +587,7 @@ fn aes256_gcm_siv_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_aws); }) }); @@ -593,10 +608,10 @@ fn aes128_gcm_siv_encrypt(c: &mut Criterion) { let nonce_rs = rscrypto::aead::Nonce96::from_bytes(NONCE_12); let cipher_rs = rscrypto::Aes128GcmSiv::new(&rscrypto::Aes128GcmSivKey::from_bytes(KEY_16)); let cipher_rc = aes_gcm_siv::Aes128GcmSiv::new(&KEY_16.into()); - let nonce_rc = aes_gcm_siv::Nonce::from_slice(&NONCE_12); + let nonce_rc = aes_gcm_siv::Nonce::from(NONCE_12); aws_lc_bench! { let aws_key = - aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM_SIV, &KEY_16).unwrap()); + aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM_SIV, &KEY_16).expect("valid AEAD benchmark operation must succeed")); } let mut g = c.benchmark_group("aes-128-gcm-siv/encrypt"); @@ -616,8 +631,12 @@ fn aes128_gcm_siv_encrypt(c: &mut Criterion) { buf.copy_from_slice(d); black_box( cipher_rc - .encrypt_in_place_detached(black_box(nonce_rc), black_box(AAD), black_box(&mut buf)) - .unwrap(), + .encrypt_inout_detached( + black_box(&nonce_rc), + black_box(AAD), + black_box(buf.as_mut_slice()).into(), + ) + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -633,7 +652,7 @@ fn aes128_gcm_siv_encrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(tag.as_ref()); black_box(&buf_aws); }) @@ -653,10 +672,10 @@ fn aes128_gcm_siv_decrypt(c: &mut Criterion) { let nonce_rs = rscrypto::aead::Nonce96::from_bytes(NONCE_12); let cipher_rs = rscrypto::Aes128GcmSiv::new(&rscrypto::Aes128GcmSivKey::from_bytes(KEY_16)); let cipher_rc = aes_gcm_siv::Aes128GcmSiv::new(&KEY_16.into()); - let nonce_rc = aes_gcm_siv::Nonce::from_slice(&NONCE_12); + let nonce_rc = aes_gcm_siv::Nonce::from(NONCE_12); aws_lc_bench! { let aws_key = - aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM_SIV, &KEY_16).unwrap()); + aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM_SIV, &KEY_16).expect("valid AEAD benchmark operation must succeed")); } let mut g = c.benchmark_group("aes-128-gcm-siv/decrypt"); @@ -664,10 +683,14 @@ fn aes128_gcm_siv_decrypt(c: &mut Criterion) { common::set_throughput(&mut g, *len); let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); let mut ct_rc = data.clone(); - let tag_rc = cipher_rc.encrypt_in_place_detached(nonce_rc, AAD, &mut ct_rc).unwrap(); + let tag_rc = cipher_rc + .encrypt_inout_detached(&nonce_rc, AAD, ct_rc.as_mut_slice().into()) + .expect("valid AEAD benchmark operation must succeed"); aws_lc_bench! { let mut ct_aws = data.clone(); @@ -677,7 +700,7 @@ fn aes128_gcm_siv_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), &mut ct_aws, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); // AWS-LC exposes detached tags for seal, but its in-place open API takes ct||tag. ct_aws.extend_from_slice(tag_aws.as_ref()); } @@ -694,7 +717,7 @@ fn aes128_gcm_siv_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -705,13 +728,13 @@ fn aes128_gcm_siv_decrypt(c: &mut Criterion) { b.iter(|| { buf_rc.copy_from_slice(ct); cipher_rc - .decrypt_in_place_detached( - black_box(nonce_rc), + .decrypt_inout_detached( + black_box(&nonce_rc), black_box(AAD), - black_box(&mut buf_rc), + black_box(buf_rc.as_mut_slice()).into(), black_box(&tag_rc), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_rc); }) }); @@ -728,7 +751,7 @@ fn aes128_gcm_siv_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_aws); }) }); @@ -752,9 +775,11 @@ fn aes256_gcm_encrypt(c: &mut Criterion) { let cipher_rc = aes_gcm::Aes256Gcm::new(&KEY_32.into()); let nonce_rc = aes_gcm::Nonce::from(NONCE_12); aws_lc_bench! { - let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM, &KEY_32).unwrap()); + let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM, &KEY_32).expect("valid AEAD benchmark operation must succeed")); } - let ring_key = ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(&ring_aead::AES_256_GCM, &KEY_32).unwrap()); + let ring_key = ring_aead::LessSafeKey::new( + ring_aead::UnboundKey::new(&ring_aead::AES_256_GCM, &KEY_32).expect("valid AEAD benchmark operation must succeed"), + ); let mut g = c.benchmark_group("aes-256-gcm/encrypt"); for (len, data) in &inputs { @@ -778,7 +803,7 @@ fn aes256_gcm_encrypt(c: &mut Criterion) { black_box(AAD), black_box(buf.as_mut_slice().into()), ) - .unwrap(), + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -794,7 +819,7 @@ fn aes256_gcm_encrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(tag.as_ref()); black_box(&buf_aws); }) @@ -811,7 +836,7 @@ fn aes256_gcm_encrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), black_box(&mut buf_ring), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(tag.as_ref()); black_box(&buf_ring); }) @@ -833,21 +858,25 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { let cipher_rc = aes_gcm::Aes256Gcm::new(&KEY_32.into()); let nonce_rc = aes_gcm::Nonce::from(NONCE_12); aws_lc_bench! { - let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM, &KEY_32).unwrap()); + let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM, &KEY_32).expect("valid AEAD benchmark operation must succeed")); } - let ring_key = ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(&ring_aead::AES_256_GCM, &KEY_32).unwrap()); + let ring_key = ring_aead::LessSafeKey::new( + ring_aead::UnboundKey::new(&ring_aead::AES_256_GCM, &KEY_32).expect("valid AEAD benchmark operation must succeed"), + ); let mut g = c.benchmark_group("aes-256-gcm/decrypt"); for (len, data) in &inputs { common::set_throughput(&mut g, *len); let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); let mut ct_rc = data.clone(); let tag_rc = cipher_rc .encrypt_inout_detached(&nonce_rc, AAD, ct_rc.as_mut_slice().into()) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); aws_lc_bench! { let mut ct_aws = data.clone(); @@ -857,7 +886,7 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), &mut ct_aws, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); // AWS-LC exposes detached tags for seal, but its in-place open API takes ct||tag. ct_aws.extend_from_slice(tag_aws.as_ref()); } @@ -869,7 +898,7 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), &mut ct_ring, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); let mut buf = ciphertext.clone(); @@ -883,7 +912,7 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -900,7 +929,7 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { black_box(buf_rc.as_mut_slice().into()), black_box(&tag_rc), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_rc); }) }); @@ -917,7 +946,7 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_aws); }) }); @@ -936,7 +965,7 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { black_box(&mut buf_ring), 0.., ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_ring); }) }); @@ -959,9 +988,11 @@ fn aes128_gcm_encrypt(c: &mut Criterion) { let cipher_rc = aes_gcm::Aes128Gcm::new(&KEY_16.into()); let nonce_rc = aes_gcm::Nonce::from(NONCE_12); aws_lc_bench! { - let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM, &KEY_16).unwrap()); + let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM, &KEY_16).expect("valid AEAD benchmark operation must succeed")); } - let ring_key = ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(&ring_aead::AES_128_GCM, &KEY_16).unwrap()); + let ring_key = ring_aead::LessSafeKey::new( + ring_aead::UnboundKey::new(&ring_aead::AES_128_GCM, &KEY_16).expect("valid AEAD benchmark operation must succeed"), + ); let mut g = c.benchmark_group("aes-128-gcm/encrypt"); for (len, data) in &inputs { @@ -985,7 +1016,7 @@ fn aes128_gcm_encrypt(c: &mut Criterion) { black_box(AAD), black_box(buf.as_mut_slice().into()), ) - .unwrap(), + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -1001,7 +1032,7 @@ fn aes128_gcm_encrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(tag.as_ref()); black_box(&buf_aws); }) @@ -1018,7 +1049,7 @@ fn aes128_gcm_encrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), black_box(&mut buf_ring), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(tag.as_ref()); black_box(&buf_ring); }) @@ -1040,21 +1071,25 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { let cipher_rc = aes_gcm::Aes128Gcm::new(&KEY_16.into()); let nonce_rc = aes_gcm::Nonce::from(NONCE_12); aws_lc_bench! { - let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM, &KEY_16).unwrap()); + let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM, &KEY_16).expect("valid AEAD benchmark operation must succeed")); } - let ring_key = ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(&ring_aead::AES_128_GCM, &KEY_16).unwrap()); + let ring_key = ring_aead::LessSafeKey::new( + ring_aead::UnboundKey::new(&ring_aead::AES_128_GCM, &KEY_16).expect("valid AEAD benchmark operation must succeed"), + ); let mut g = c.benchmark_group("aes-128-gcm/decrypt"); for (len, data) in &inputs { common::set_throughput(&mut g, *len); let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); let mut ct_rc = data.clone(); let tag_rc = cipher_rc .encrypt_inout_detached(&nonce_rc, AAD, ct_rc.as_mut_slice().into()) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); aws_lc_bench! { let mut ct_aws = data.clone(); @@ -1064,7 +1099,7 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), &mut ct_aws, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); // AWS-LC exposes detached tags for seal, but its in-place open API takes ct||tag. ct_aws.extend_from_slice(tag_aws.as_ref()); } @@ -1076,7 +1111,7 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), &mut ct_ring, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); let mut buf = ciphertext.clone(); @@ -1090,7 +1125,7 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -1107,7 +1142,7 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { black_box(buf_rc.as_mut_slice().into()), black_box(&tag_rc), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_rc); }) }); @@ -1124,7 +1159,7 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_aws); }) }); @@ -1143,7 +1178,7 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { black_box(&mut buf_ring), 0.., ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_ring); }) }); @@ -1194,7 +1229,9 @@ fn aegis256_decrypt(c: &mut Criterion) { // Pre-encrypt with rscrypto to get valid ciphertext + tag. let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); // Pre-encrypt with aegis crate to get its tag format. let mut ct_ac = data.clone(); @@ -1213,7 +1250,7 @@ fn aegis256_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -1225,7 +1262,7 @@ fn aegis256_decrypt(c: &mut Criterion) { buf_ac.copy_from_slice(ct); cipher_ac .decrypt_in_place(black_box(&mut buf_ac), black_box(&tag_ac), black_box(AAD)) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_ac); }) }); @@ -1267,7 +1304,7 @@ fn ascon_aead128_encrypt(c: &mut Criterion) { black_box(AAD), black_box(buf.as_mut_slice().into()), ) - .unwrap(), + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -1290,12 +1327,14 @@ fn ascon_aead128_decrypt(c: &mut Criterion) { common::set_throughput(&mut g, *len); let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); let mut ct_ac = data.clone(); let tag_ac = cipher_ac .encrypt_inout_detached(&nonce_ac, AAD, ct_ac.as_mut_slice().into()) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); let mut buf = ciphertext.clone(); @@ -1309,7 +1348,7 @@ fn ascon_aead128_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -1326,7 +1365,7 @@ fn ascon_aead128_decrypt(c: &mut Criterion) { black_box(buf_ac.as_mut_slice().into()), black_box(&tag_ac), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_ac); }) }); diff --git a/benches/aead_kernels.rs b/benches/aead_kernels.rs index 4ecc827f..17108f67 100644 --- a/benches/aead_kernels.rs +++ b/benches/aead_kernels.rs @@ -31,12 +31,16 @@ fn chacha20_xor_kernel(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("aarch64-neon", len), data, |b, d| { b.iter(|| { buf.copy_from_slice(d); - rscrypto::aead::diag_chacha20_xor_keystream_aarch64_neon( - black_box(&KEY_32), - black_box(1), - black_box(&NONCE_12), - black_box(&mut buf), - ); + // SAFETY: AArch64 NEON is part of this target's compile-time baseline. `comp_sizes` tops out at 1,048,576 + // bytes (16,384 blocks), which fits the `u32` counter range starting at 1. + unsafe { + rscrypto::aead::diag_chacha20_xor_keystream_aarch64_neon( + black_box(&KEY_32), + black_box(1), + black_box(&NONCE_12), + black_box(&mut buf), + ); + } black_box(buf.as_ptr()) }) }); @@ -59,7 +63,7 @@ fn poly1305_auth_kernel(c: &mut Criterion) { b.iter(|| { black_box( rscrypto::aead::diag_chacha20poly1305_authenticate_aead(black_box(AAD), black_box(d), black_box(&POLY_KEY)) - .unwrap(), + .expect("selected AEAD benchmark kernel must be available"), ) }) }); @@ -73,7 +77,7 @@ fn poly1305_auth_kernel(c: &mut Criterion) { black_box(d), black_box(&POLY_KEY), ) - .unwrap(), + .expect("selected AEAD benchmark kernel must be available"), ) }) }); diff --git a/benches/ascon.rs b/benches/ascon.rs index 12de7ea3..bdf789a6 100644 --- a/benches/ascon.rs +++ b/benches/ascon.rs @@ -140,8 +140,8 @@ fn ascon_xof128_many(c: &mut Criterion) { let mut hasher = rscrypto::AsconXof::new(); hasher.update(black_box(input)); let mut reader = hasher.finalize_xof(); - let base = index * OUT_LEN; - reader.squeeze(&mut out[base..base + OUT_LEN]); + let base = index.strict_mul(OUT_LEN); + reader.squeeze(&mut out[base..base.strict_add(OUT_LEN)]); } black_box(out[0]) }) @@ -163,7 +163,8 @@ fn ascon_cxof128(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rscrypto", len), data, |b, d| { b.iter(|| { let mut out = [0u8; OUT_LEN]; - rscrypto::AsconCxof128::hash_into(black_box(CUSTOMIZATION), black_box(d), &mut out).unwrap(); + rscrypto::AsconCxof128::hash_into(black_box(CUSTOMIZATION), black_box(d), &mut out) + .expect("valid Ascon benchmark operation must succeed"); black_box(out) }) }); diff --git a/benches/auth.rs b/benches/auth.rs index 9a37ac57..d16956c4 100644 --- a/benches/auth.rs +++ b/benches/auth.rs @@ -43,7 +43,8 @@ fn array_from_slice(slice: &[u8]) -> [u8; N] { fn deterministic_bytes(offset: u8) -> [u8; N] { let mut out = [0u8; N]; for (i, byte) in out.iter_mut().enumerate() { - *byte = offset.wrapping_add(i as u8); + let low_index = u8::try_from(i & usize::from(u8::MAX)).expect("masked deterministic-byte index must fit u8"); + *byte = offset.wrapping_add(low_index); } out } @@ -147,7 +148,8 @@ fn hmac_sha256(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto", len), data, |b, d| { - let base_mac = RustCryptoHmacSha256::new_from_slice(&key).unwrap(); + let base_mac = + RustCryptoHmacSha256::new_from_slice(&key).expect("valid authentication benchmark operation must succeed"); b.iter(|| { use hmac::Mac as _; @@ -196,7 +198,8 @@ fn hmac_sha384(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto", len), data, |b, d| { - let base_mac = RustCryptoHmacSha384::new_from_slice(&key).unwrap(); + let base_mac = + RustCryptoHmacSha384::new_from_slice(&key).expect("valid authentication benchmark operation must succeed"); b.iter(|| { use hmac::Mac as _; @@ -245,7 +248,8 @@ fn hmac_sha512(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto", len), data, |b, d| { - let base_mac = RustCryptoHmacSha512::new_from_slice(&key).unwrap(); + let base_mac = + RustCryptoHmacSha512::new_from_slice(&key).expect("valid authentication benchmark operation must succeed"); b.iter(|| { use hmac::Mac as _; @@ -292,7 +296,8 @@ fn hmac_sha256_streaming(c: &mut Criterion) { b.iter(|| { use hmac::Mac as _; - let mut mac = RustCryptoHmacSha256::new_from_slice(&key).unwrap(); + let mut mac = + RustCryptoHmacSha256::new_from_slice(&key).expect("valid authentication benchmark operation must succeed"); for chunk in data.chunks(chunk_size) { mac.update(black_box(chunk)); } @@ -345,7 +350,8 @@ fn hmac_sha256_internal(c: &mut Criterion) { b.iter(|| { use hmac::Mac as _; - let mut mac = RustCryptoHmacSha256::new_from_slice(black_box(&key)).unwrap(); + let mut mac = RustCryptoHmacSha256::new_from_slice(black_box(&key)) + .expect("valid authentication benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -384,7 +390,9 @@ fn hkdf_sha256_expand(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rscrypto", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - hkdf.expand(black_box(&info), black_box(&mut out)).unwrap(); + hkdf + .expand(black_box(&info), black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -392,7 +400,9 @@ fn hkdf_sha256_expand(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rustcrypto", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - rustcrypto.expand(black_box(&info), black_box(&mut out)).unwrap(); + rustcrypto + .expand(black_box(&info), black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -403,9 +413,9 @@ fn hkdf_sha256_expand(c: &mut Criterion) { b.iter(|| { aws_prk .expand(&[black_box(&info)], AwsHkdfLen(len)) - .unwrap() + .expect("valid authentication benchmark operation must succeed") .fill(black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -416,9 +426,9 @@ fn hkdf_sha256_expand(c: &mut Criterion) { b.iter(|| { ring_prk .expand(&[black_box(&info)], RingHkdfLen(len)) - .unwrap() + .expect("valid authentication benchmark operation must succeed") .fill(black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -446,7 +456,9 @@ fn hkdf_sha384_expand(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rscrypto", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - hkdf.expand(black_box(&info), black_box(&mut out)).unwrap(); + hkdf + .expand(black_box(&info), black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -454,7 +466,9 @@ fn hkdf_sha384_expand(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rustcrypto", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - rustcrypto.expand(black_box(&info), black_box(&mut out)).unwrap(); + rustcrypto + .expand(black_box(&info), black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -465,9 +479,9 @@ fn hkdf_sha384_expand(c: &mut Criterion) { b.iter(|| { aws_prk .expand(&[black_box(&info)], AwsHkdfLen(len)) - .unwrap() + .expect("valid authentication benchmark operation must succeed") .fill(black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -478,9 +492,9 @@ fn hkdf_sha384_expand(c: &mut Criterion) { b.iter(|| { ring_prk .expand(&[black_box(&info)], RingHkdfLen(len)) - .unwrap() + .expect("valid authentication benchmark operation must succeed") .fill(black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -497,7 +511,8 @@ fn pbkdf2_sha256_derive(c: &mut Criterion) { let state = Pbkdf2Sha256::new(&password); for &iterations in &[1u32, 100, 1000] { - let nz_iters = core::num::NonZeroU32::new(iterations).unwrap(); + let nz_iters = + core::num::NonZeroU32::new(iterations).expect("valid authentication benchmark operation must succeed"); let mut g = c.benchmark_group(format!("pbkdf2-sha256/iters={iterations}")); for &out_len in &[32usize, 64] { @@ -507,7 +522,7 @@ fn pbkdf2_sha256_derive(c: &mut Criterion) { let mut out = vec![0u8; len]; b.iter(|| { Pbkdf2Sha256::derive_key_primitive(black_box(&password), black_box(&salt), iterations, black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -559,7 +574,9 @@ fn pbkdf2_sha256_derive(c: &mut Criterion) { g_state.bench_with_input(BenchmarkId::new("rscrypto", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - state.derive(black_box(&salt), iterations, black_box(&mut out)).unwrap(); + state + .derive(black_box(&salt), iterations, black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -576,7 +593,8 @@ fn pbkdf2_sha256_internal(c: &mut Criterion) { let state = Pbkdf2Sha256::new(&password); for &iterations in &[1u32, 100, 1000] { - let nz_iters = core::num::NonZeroU32::new(iterations).unwrap(); + let nz_iters = + core::num::NonZeroU32::new(iterations).expect("valid authentication benchmark operation must succeed"); let mut g = c.benchmark_group(format!("pbkdf2-sha256/internal/iters={iterations}")); for &out_len in &[32usize, 64] { @@ -586,7 +604,7 @@ fn pbkdf2_sha256_internal(c: &mut Criterion) { let mut out = vec![0u8; len]; b.iter(|| { Pbkdf2Sha256::derive_key_primitive(black_box(&password), black_box(&salt), iterations, black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -594,7 +612,9 @@ fn pbkdf2_sha256_internal(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rscrypto-state", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - state.derive(black_box(&salt), iterations, black_box(&mut out)).unwrap(); + state + .derive(black_box(&salt), iterations, black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -650,7 +670,8 @@ fn pbkdf2_sha512_derive(c: &mut Criterion) { let state = Pbkdf2Sha512::new(&password); for &iterations in &[1u32, 100, 1000] { - let nz_iters = core::num::NonZeroU32::new(iterations).unwrap(); + let nz_iters = + core::num::NonZeroU32::new(iterations).expect("valid authentication benchmark operation must succeed"); let mut g = c.benchmark_group(format!("pbkdf2-sha512/iters={iterations}")); for &out_len in &[64usize, 128] { @@ -660,7 +681,7 @@ fn pbkdf2_sha512_derive(c: &mut Criterion) { let mut out = vec![0u8; len]; b.iter(|| { Pbkdf2Sha512::derive_key_primitive(black_box(&password), black_box(&salt), iterations, black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -712,7 +733,9 @@ fn pbkdf2_sha512_derive(c: &mut Criterion) { g_state.bench_with_input(BenchmarkId::new("rscrypto", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - state.derive(black_box(&salt), iterations, black_box(&mut out)).unwrap(); + state + .derive(black_box(&salt), iterations, black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -744,13 +767,15 @@ fn ed25519_public_key(c: &mut Criterion) { fn ecdsa_p256_verify(c: &mut Criterion) { let secret_bytes = [0x11u8; 32]; - let signing_key = P256OracleSigningKey::from_slice(&secret_bytes).unwrap(); + let signing_key = + P256OracleSigningKey::from_slice(&secret_bytes).expect("valid authentication benchmark operation must succeed"); let verifying_key = signing_key.verifying_key(); let sec1 = EcdsaP256SecretKey::from_bytes(secret_bytes) - .unwrap() + .expect("valid authentication benchmark operation must succeed") .public_key() .to_sec1_bytes(); - let public = EcdsaP256PublicKey::from_sec1_bytes(sec1.as_slice()).unwrap(); + let public = EcdsaP256PublicKey::from_sec1_bytes(sec1.as_slice()) + .expect("valid authentication benchmark operation must succeed"); let ring_upk = ring::signature::UnparsedPublicKey::new(&ring::signature::ECDSA_P256_SHA256_FIXED, sec1.as_slice()); aws_lc_bench! { let aws_upk = @@ -766,11 +791,14 @@ fn ecdsa_p256_verify(c: &mut Criterion) { for (len, data) in &inputs { common::set_throughput(&mut g, *len); let oracle_signature: P256OracleSignature = p256::ecdsa::signature::Signer::sign(&signing_key, data); - let signature = EcdsaP256Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())).unwrap(); + let signature = EcdsaP256Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())) + .expect("valid authentication benchmark operation must succeed"); g.bench_with_input(BenchmarkId::new("rscrypto", len), data, |b, d| { b.iter(|| { - black_box(&public).verify(black_box(d), black_box(&signature)).unwrap(); + black_box(&public) + .verify(black_box(d), black_box(&signature)) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -778,14 +806,16 @@ fn ecdsa_p256_verify(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rustcrypto-p256", len), data, |b, d| { b.iter(|| { p256::ecdsa::signature::Verifier::verify(black_box(verifying_key), black_box(d), black_box(&oracle_signature)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); g.bench_with_input(BenchmarkId::new("ring", len), data, |b, d| { b.iter(|| { - ring_upk.verify(black_box(d), black_box(signature.as_bytes())).unwrap(); + ring_upk + .verify(black_box(d), black_box(signature.as_bytes())) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -793,7 +823,7 @@ fn ecdsa_p256_verify(c: &mut Criterion) { aws_lc_bench! { g.bench_with_input(BenchmarkId::new("aws-lc-rs", len), data, |b, d| { b.iter(|| { - aws_upk.verify(black_box(d), black_box(signature.as_bytes())).unwrap(); + aws_upk.verify(black_box(d), black_box(signature.as_bytes())).expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -805,10 +835,12 @@ fn ecdsa_p256_verify(c: &mut Criterion) { fn ecdsa_p256_sign(c: &mut Criterion) { let secret_bytes = [0x11u8; 32]; - let secret = EcdsaP256SecretKey::from_bytes(secret_bytes).unwrap(); + let secret = + EcdsaP256SecretKey::from_bytes(secret_bytes).expect("valid authentication benchmark operation must succeed"); let keypair = EcdsaP256Keypair::from_secret_key(secret); let blind = [0x5cu8; 64]; - let signing_key = P256OracleSigningKey::from_slice(&secret_bytes).unwrap(); + let signing_key = + P256OracleSigningKey::from_slice(&secret_bytes).expect("valid authentication benchmark operation must succeed"); let sec1 = keypair.public_key().to_sec1_bytes(); let ring_rng = ring::rand::SystemRandom::new(); let ring_key = ring::signature::EcdsaKeyPair::from_private_key_and_public_key( @@ -817,7 +849,7 @@ fn ecdsa_p256_sign(c: &mut Criterion) { &sec1, &ring_rng, ) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); aws_lc_bench! { let aws_rng = aws_lc_rs::rand::SystemRandom::new(); let aws_key = aws_lc_rs::signature::EcdsaKeyPair::from_private_key_and_public_key( @@ -825,7 +857,7 @@ fn ecdsa_p256_sign(c: &mut Criterion) { &secret_bytes, &sec1, ) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); } let inputs = [0usize, 32, 1024, 16384] @@ -838,7 +870,13 @@ fn ecdsa_p256_sign(c: &mut Criterion) { common::set_throughput(&mut g, *len); g.bench_with_input(BenchmarkId::new("rscrypto-deterministic", len), data, |b, d| { - b.iter(|| black_box(black_box(&keypair).try_sign(black_box(d)).unwrap())) + b.iter(|| { + black_box( + black_box(&keypair) + .try_sign(black_box(d)) + .expect("valid authentication benchmark operation must succeed"), + ) + }) }); g.bench_with_input(BenchmarkId::new("rscrypto-blinded", len), data, |b, d| { @@ -846,7 +884,7 @@ fn ecdsa_p256_sign(c: &mut Criterion) { black_box( black_box(&keypair) .try_sign_blinded(black_box(d), |out| out.copy_from_slice(black_box(&blind))) - .unwrap(), + .expect("valid authentication benchmark operation must succeed"), ) }) }); @@ -860,12 +898,18 @@ fn ecdsa_p256_sign(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("ring", len), data, |b, d| { - b.iter(|| black_box(ring_key.sign(&ring_rng, black_box(d)).unwrap())) + b.iter(|| { + black_box( + ring_key + .sign(&ring_rng, black_box(d)) + .expect("valid authentication benchmark operation must succeed"), + ) + }) }); aws_lc_bench! { g.bench_with_input(BenchmarkId::new("aws-lc-rs", len), data, |b, d| { - b.iter(|| black_box(aws_key.sign(&aws_rng, black_box(d)).unwrap())) + b.iter(|| black_box(aws_key.sign(&aws_rng, black_box(d)).expect("valid authentication benchmark operation must succeed"))) }); } } @@ -875,13 +919,15 @@ fn ecdsa_p256_sign(c: &mut Criterion) { fn ecdsa_p384_verify(c: &mut Criterion) { let secret_bytes = [0x31u8; 48]; - let signing_key = P384OracleSigningKey::from_slice(&secret_bytes).unwrap(); + let signing_key = + P384OracleSigningKey::from_slice(&secret_bytes).expect("valid authentication benchmark operation must succeed"); let verifying_key = signing_key.verifying_key(); let sec1 = EcdsaP384SecretKey::from_bytes(secret_bytes) - .unwrap() + .expect("valid authentication benchmark operation must succeed") .public_key() .to_sec1_bytes(); - let public = EcdsaP384PublicKey::from_sec1_bytes(sec1.as_slice()).unwrap(); + let public = EcdsaP384PublicKey::from_sec1_bytes(sec1.as_slice()) + .expect("valid authentication benchmark operation must succeed"); let ring_upk = ring::signature::UnparsedPublicKey::new(&ring::signature::ECDSA_P384_SHA384_FIXED, sec1.as_slice()); aws_lc_bench! { let aws_upk = @@ -897,11 +943,14 @@ fn ecdsa_p384_verify(c: &mut Criterion) { for (len, data) in &inputs { common::set_throughput(&mut g, *len); let oracle_signature: P384OracleSignature = p384::ecdsa::signature::Signer::sign(&signing_key, data); - let signature = EcdsaP384Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())).unwrap(); + let signature = EcdsaP384Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())) + .expect("valid authentication benchmark operation must succeed"); g.bench_with_input(BenchmarkId::new("rscrypto", len), data, |b, d| { b.iter(|| { - black_box(&public).verify(black_box(d), black_box(&signature)).unwrap(); + black_box(&public) + .verify(black_box(d), black_box(&signature)) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -909,14 +958,16 @@ fn ecdsa_p384_verify(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rustcrypto-p384", len), data, |b, d| { b.iter(|| { p384::ecdsa::signature::Verifier::verify(black_box(verifying_key), black_box(d), black_box(&oracle_signature)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); g.bench_with_input(BenchmarkId::new("ring", len), data, |b, d| { b.iter(|| { - ring_upk.verify(black_box(d), black_box(signature.as_bytes())).unwrap(); + ring_upk + .verify(black_box(d), black_box(signature.as_bytes())) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -924,7 +975,7 @@ fn ecdsa_p384_verify(c: &mut Criterion) { aws_lc_bench! { g.bench_with_input(BenchmarkId::new("aws-lc-rs", len), data, |b, d| { b.iter(|| { - aws_upk.verify(black_box(d), black_box(signature.as_bytes())).unwrap(); + aws_upk.verify(black_box(d), black_box(signature.as_bytes())).expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -936,10 +987,12 @@ fn ecdsa_p384_verify(c: &mut Criterion) { fn ecdsa_p384_sign(c: &mut Criterion) { let secret_bytes = [0x31u8; 48]; - let secret = EcdsaP384SecretKey::from_bytes(secret_bytes).unwrap(); + let secret = + EcdsaP384SecretKey::from_bytes(secret_bytes).expect("valid authentication benchmark operation must succeed"); let keypair = EcdsaP384Keypair::from_secret_key(secret); let blind = [0xa3u8; 96]; - let signing_key = P384OracleSigningKey::from_slice(&secret_bytes).unwrap(); + let signing_key = + P384OracleSigningKey::from_slice(&secret_bytes).expect("valid authentication benchmark operation must succeed"); let sec1 = keypair.public_key().to_sec1_bytes(); let ring_rng = ring::rand::SystemRandom::new(); let ring_key = ring::signature::EcdsaKeyPair::from_private_key_and_public_key( @@ -948,7 +1001,7 @@ fn ecdsa_p384_sign(c: &mut Criterion) { &sec1, &ring_rng, ) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); aws_lc_bench! { let aws_rng = aws_lc_rs::rand::SystemRandom::new(); let aws_key = aws_lc_rs::signature::EcdsaKeyPair::from_private_key_and_public_key( @@ -956,7 +1009,7 @@ fn ecdsa_p384_sign(c: &mut Criterion) { &secret_bytes, &sec1, ) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); } let inputs = [0usize, 32, 1024, 16384] @@ -969,7 +1022,13 @@ fn ecdsa_p384_sign(c: &mut Criterion) { common::set_throughput(&mut g, *len); g.bench_with_input(BenchmarkId::new("rscrypto-deterministic", len), data, |b, d| { - b.iter(|| black_box(black_box(&keypair).try_sign(black_box(d)).unwrap())) + b.iter(|| { + black_box( + black_box(&keypair) + .try_sign(black_box(d)) + .expect("valid authentication benchmark operation must succeed"), + ) + }) }); g.bench_with_input(BenchmarkId::new("rscrypto-blinded", len), data, |b, d| { @@ -977,7 +1036,7 @@ fn ecdsa_p384_sign(c: &mut Criterion) { black_box( black_box(&keypair) .try_sign_blinded(black_box(d), |out| out.copy_from_slice(black_box(&blind))) - .unwrap(), + .expect("valid authentication benchmark operation must succeed"), ) }) }); @@ -991,12 +1050,18 @@ fn ecdsa_p384_sign(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("ring", len), data, |b, d| { - b.iter(|| black_box(ring_key.sign(&ring_rng, black_box(d)).unwrap())) + b.iter(|| { + black_box( + ring_key + .sign(&ring_rng, black_box(d)) + .expect("valid authentication benchmark operation must succeed"), + ) + }) }); aws_lc_bench! { g.bench_with_input(BenchmarkId::new("aws-lc-rs", len), data, |b, d| { - b.iter(|| black_box(aws_key.sign(&aws_rng, black_box(d)).unwrap())) + b.iter(|| black_box(aws_key.sign(&aws_rng, black_box(d)).expect("valid authentication benchmark operation must succeed"))) }); } } @@ -1118,9 +1183,10 @@ fn ed25519_sign(c: &mut Criterion) { let keypair = Ed25519Keypair::from_secret_key(secret.duplicate_secret()); let signing_key = SigningKey::from_bytes(&secret_bytes); aws_lc_bench! { - let aws_kp = aws_lc_rs::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes).unwrap(); + let aws_kp = aws_lc_rs::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes).expect("valid authentication benchmark operation must succeed"); } - let ring_kp = ring::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes).unwrap(); + let ring_kp = ring::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes) + .expect("valid authentication benchmark operation must succeed"); let (_dryoc_pk, dryoc_sk) = crypto_sign_seed_keypair(&secret_bytes); let mut dryoc_sig: [u8; 64] = [0u8; 64]; let inputs = [0usize, 32, 1024, 16384] @@ -1156,7 +1222,8 @@ fn ed25519_sign(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("dryoc", len), data, |b, d| { b.iter(|| { - crypto_sign_detached(&mut dryoc_sig, black_box(d), &dryoc_sk).unwrap(); + crypto_sign_detached(&mut dryoc_sig, black_box(d), &dryoc_sk) + .expect("valid authentication benchmark operation must succeed"); black_box(&dryoc_sig); }) }); @@ -1179,11 +1246,12 @@ fn ed25519_verify(c: &mut Criterion) { let signing_key = SigningKey::from_bytes(&secret_bytes); let verifying_key = signing_key.verifying_key(); aws_lc_bench! { - let aws_kp = aws_lc_rs::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes).unwrap(); + let aws_kp = aws_lc_rs::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes).expect("valid authentication benchmark operation must succeed"); let aws_pubkey: Vec = aws_kp.public_key().as_ref().to_vec(); let aws_upk = aws_lc_rs::signature::UnparsedPublicKey::new(&aws_lc_rs::signature::ED25519, aws_pubkey); } - let ring_kp = ring::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes).unwrap(); + let ring_kp = ring::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes) + .expect("valid authentication benchmark operation must succeed"); let ring_pubkey: Vec = ring_kp.public_key().as_ref().to_vec(); let ring_upk = ring::signature::UnparsedPublicKey::new(&ring::signature::ED25519, ring_pubkey); let (dryoc_pk, dryoc_sk) = crypto_sign_seed_keypair(&secret_bytes); @@ -1202,11 +1270,14 @@ fn ed25519_verify(c: &mut Criterion) { } let ring_sig = ring_kp.sign(data); let mut dryoc_sig: [u8; 64] = [0u8; 64]; - crypto_sign_detached(&mut dryoc_sig, data, &dryoc_sk).unwrap(); + crypto_sign_detached(&mut dryoc_sig, data, &dryoc_sk) + .expect("valid authentication benchmark operation must succeed"); g.bench_with_input(BenchmarkId::new("rscrypto", len), data, |b, d| { b.iter(|| { - black_box(&public).verify(black_box(d), black_box(&ours)).unwrap(); + black_box(&public) + .verify(black_box(d), black_box(&ours)) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -1215,7 +1286,7 @@ fn ed25519_verify(c: &mut Criterion) { b.iter(|| { black_box(&verifying_key) .verify_strict(black_box(d), black_box(&dalek)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -1223,7 +1294,7 @@ fn ed25519_verify(c: &mut Criterion) { aws_lc_bench! { g.bench_with_input(BenchmarkId::new("aws-lc-rs", len), data, |b, d| { b.iter(|| { - aws_upk.verify(black_box(d), aws_sig.as_ref()).unwrap(); + aws_upk.verify(black_box(d), aws_sig.as_ref()).expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -1231,14 +1302,17 @@ fn ed25519_verify(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("ring", len), data, |b, d| { b.iter(|| { - ring_upk.verify(black_box(d), ring_sig.as_ref()).unwrap(); + ring_upk + .verify(black_box(d), ring_sig.as_ref()) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); g.bench_with_input(BenchmarkId::new("dryoc", len), data, |b, d| { b.iter(|| { - crypto_sign_verify_detached(&dryoc_sig, black_box(d), &dryoc_pk).unwrap(); + crypto_sign_verify_detached(&dryoc_sig, black_box(d), &dryoc_pk) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -1269,7 +1343,8 @@ fn ed25519_verify_phase(c: &mut Criterion) { for (len, data) in &inputs { common::set_throughput(&mut g, *len); let signature = keypair.sign(data); - let scalars = diag_ed25519_verify_scalars(&public, &signature, data).unwrap(); + let scalars = diag_ed25519_verify_scalars(&public, &signature, data) + .expect("valid authentication benchmark operation must succeed"); g.bench_with_input(BenchmarkId::new("challenge-reduce", len), data, |b, d| { b.iter(|| { @@ -1352,8 +1427,8 @@ fn x25519_public_key(c: &mut Criterion) { b.iter(|| { let priv_key = aws_lc_rs::agreement::PrivateKey::from_private_key(&aws_lc_rs::agreement::X25519, black_box(&secret_bytes)) - .unwrap(); - black_box(priv_key.compute_public_key().unwrap()) + .expect("valid authentication benchmark operation must succeed"); + black_box(priv_key.compute_public_key().expect("valid authentication benchmark operation must succeed")) }) }); } @@ -1381,11 +1456,11 @@ fn x25519_diffie_hellman(c: &mut Criterion) { let dalek_bob_public = DalekX25519PublicKey::from(&DalekX25519Secret::from(bob_bytes)); aws_lc_bench! { let aws_alice = - aws_lc_rs::agreement::PrivateKey::from_private_key(&aws_lc_rs::agreement::X25519, &alice_bytes).unwrap(); + aws_lc_rs::agreement::PrivateKey::from_private_key(&aws_lc_rs::agreement::X25519, &alice_bytes).expect("valid authentication benchmark operation must succeed"); let aws_bob_pub_bytes: [u8; 32] = { let bob_priv = - aws_lc_rs::agreement::PrivateKey::from_private_key(&aws_lc_rs::agreement::X25519, &bob_bytes).unwrap(); - let pk = bob_priv.compute_public_key().unwrap(); + aws_lc_rs::agreement::PrivateKey::from_private_key(&aws_lc_rs::agreement::X25519, &bob_bytes).expect("valid authentication benchmark operation must succeed"); + let pk = bob_priv.compute_public_key().expect("valid authentication benchmark operation must succeed"); let mut out = [0u8; 32]; out.copy_from_slice(pk.as_ref()); out @@ -1398,7 +1473,13 @@ fn x25519_diffie_hellman(c: &mut Criterion) { let mut g = c.benchmark_group("x25519/diffie-hellman"); g.bench_function("rscrypto", |b| { - b.iter(|| black_box(black_box(&alice).diffie_hellman(black_box(&bob_public)).unwrap())) + b.iter(|| { + black_box( + black_box(&alice) + .diffie_hellman(black_box(&bob_public)) + .expect("valid authentication benchmark operation must succeed"), + ) + }) }); g.bench_function("dalek", |b| { @@ -1413,7 +1494,7 @@ fn x25519_diffie_hellman(c: &mut Criterion) { out.copy_from_slice(bytes); Ok::<[u8; 32], ()>(out) }) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(shared) }) }); @@ -1422,7 +1503,8 @@ fn x25519_diffie_hellman(c: &mut Criterion) { g.bench_function("dryoc", |b| { let mut shared = [0u8; 32]; b.iter(|| { - crypto_scalarmult(&mut shared, black_box(&alice_bytes), black_box(&dryoc_bob_pub)); + crypto_scalarmult(&mut shared, black_box(&alice_bytes), black_box(&dryoc_bob_pub)) + .expect("valid authentication benchmark operation must succeed"); black_box(shared) }) }); @@ -1454,7 +1536,7 @@ macro_rules! mlkem_profile_benches { out.copy_from_slice(black_box(&key_random)); Ok::<(), MlKemError>(()) }) - .unwrap() + .expect("valid authentication benchmark operation must succeed") }) }); @@ -1464,7 +1546,7 @@ macro_rules! mlkem_profile_benches { aws_lc_bench! { g.bench_function("aws-lc-rs", |b| { - b.iter(|| black_box(AwsMlKemDecapsulationKey::generate(&$aws_algorithm).unwrap())) + b.iter(|| black_box(AwsMlKemDecapsulationKey::generate(&$aws_algorithm).expect("valid authentication benchmark operation must succeed"))) }); } @@ -1490,8 +1572,8 @@ macro_rules! mlkem_profile_benches { out.copy_from_slice(&key_random); Ok::<(), MlKemError>(()) }) - .unwrap(); - let prepared_ek = ek.prepare().unwrap(); + .expect("valid authentication benchmark operation must succeed"); + let prepared_ek = ek.prepare().expect("valid authentication benchmark operation must succeed"); let (fips_ek, _) = $fips::KG::keygen_from_seed( array_from_slice::<32>(&key_random[..32]), array_from_slice::<32>(&key_random[32..]), @@ -1502,8 +1584,8 @@ macro_rules! mlkem_profile_benches { let libcrux_keypair = $libcrux::generate_key_pair(key_random); let libcrux_ek = libcrux_keypair.public_key().clone(); aws_lc_bench! { - let aws_dk = AwsMlKemDecapsulationKey::generate(&$aws_algorithm).unwrap(); - let aws_ek = aws_dk.encapsulation_key().unwrap(); + let aws_dk = AwsMlKemDecapsulationKey::generate(&$aws_algorithm).expect("valid authentication benchmark operation must succeed"); + let aws_ek = aws_dk.encapsulation_key().expect("valid authentication benchmark operation must succeed"); } let mut g = c.benchmark_group(concat!($group, "/encapsulate")); @@ -1514,7 +1596,7 @@ macro_rules! mlkem_profile_benches { out.copy_from_slice(black_box(&encaps_random)); Ok::<(), MlKemError>(()) }) - .unwrap() + .expect("valid authentication benchmark operation must succeed") }) }); @@ -1524,7 +1606,7 @@ macro_rules! mlkem_profile_benches { aws_lc_bench! { g.bench_function("aws-lc-rs", |b| { - b.iter(|| black_box(aws_ek.encapsulate().unwrap())) + b.iter(|| black_box(aws_ek.encapsulate().expect("valid authentication benchmark operation must succeed"))) }); } @@ -1548,15 +1630,15 @@ macro_rules! mlkem_profile_benches { out.copy_from_slice(&key_random); Ok::<(), MlKemError>(()) }) - .unwrap(); - let prepared_ek = ek.prepare().unwrap(); - let prepared_dk = dk.prepare().unwrap(); + .expect("valid authentication benchmark operation must succeed"); + let prepared_ek = ek.prepare().expect("valid authentication benchmark operation must succeed"); + let prepared_dk = dk.prepare().expect("valid authentication benchmark operation must succeed"); let (ciphertext, _) = prepared_ek .encapsulate(|out| { out.copy_from_slice(&encaps_random); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); let (fips_ek, fips_dk) = $fips::KG::keygen_from_seed( array_from_slice::<32>(&key_random[..32]), array_from_slice::<32>(&key_random[32..]), @@ -1572,14 +1654,14 @@ macro_rules! mlkem_profile_benches { let libcrux_dk = libcrux_keypair.private_key().clone(); let (libcrux_ciphertext, _) = $libcrux::encapsulate(&libcrux_ek, encaps_random); aws_lc_bench! { - let aws_dk = AwsMlKemDecapsulationKey::generate(&$aws_algorithm).unwrap(); - let aws_ek = aws_dk.encapsulation_key().unwrap(); - let (aws_ciphertext, _) = aws_ek.encapsulate().unwrap(); + let aws_dk = AwsMlKemDecapsulationKey::generate(&$aws_algorithm).expect("valid authentication benchmark operation must succeed"); + let aws_ek = aws_dk.encapsulation_key().expect("valid authentication benchmark operation must succeed"); + let (aws_ciphertext, _) = aws_ek.encapsulate().expect("valid authentication benchmark operation must succeed"); } let mut g = c.benchmark_group(concat!($group, "/decapsulate")); g.bench_function("rscrypto", |b| { - b.iter(|| black_box(&prepared_dk).decapsulate(black_box(&ciphertext)).unwrap()) + b.iter(|| black_box(&prepared_dk).decapsulate(black_box(&ciphertext)).expect("valid authentication benchmark operation must succeed")) }); g.bench_function("libcrux", |b| { @@ -1597,14 +1679,14 @@ macro_rules! mlkem_profile_benches { black_box( aws_dk .decapsulate(AwsMlKemCiphertext::from(black_box(aws_ciphertext.as_ref()))) - .unwrap(), + .expect("valid authentication benchmark operation must succeed"), ) }) }); } g.bench_function("fips203", |b| { - b.iter(|| black_box(fips_dk.try_decaps(black_box(&fips_ciphertext)).unwrap())) + b.iter(|| black_box(fips_dk.try_decaps(black_box(&fips_ciphertext)).expect("valid authentication benchmark operation must succeed"))) }); g.bench_function("rustcrypto", |b| { diff --git a/benches/blake2.rs b/benches/blake2.rs index 7e851d9e..bf31d82b 100644 --- a/benches/blake2.rs +++ b/benches/blake2.rs @@ -5,12 +5,14 @@ mod common; use core::hint::black_box; use blake2::{ - Blake2b256 as RustCryptoBlake2b256, Blake2b512 as RustCryptoBlake2b512, Blake2bMac, - Blake2s128 as RustCryptoBlake2s128, Blake2s256 as RustCryptoBlake2s256, Blake2sMac, Digest as _, + Blake2b as RustCryptoBlake2b, Blake2b512 as RustCryptoBlake2b512, Blake2bMac, Blake2s as RustCryptoBlake2s, + Blake2s256 as RustCryptoBlake2s256, Blake2sMac, + digest::{ + Digest as _, Mac as _, + consts::{U16, U32, U64}, + }, }; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; -use digest::typenum::{U16, U32, U64}; -use hmac::{Mac as _, digest::KeyInit}; use rscrypto::{ Blake2b256, Blake2b512, Blake2bKey, Blake2bParams, Blake2s128, Blake2s256, Blake2sKey, Blake2sParams, Digest, }; @@ -19,6 +21,8 @@ type RustCryptoBlake2bMac256 = Blake2bMac; type RustCryptoBlake2bMac512 = Blake2bMac; type RustCryptoBlake2sMac128 = Blake2sMac; type RustCryptoBlake2sMac256 = Blake2sMac; +type RustCryptoBlake2b256 = RustCryptoBlake2b; +type RustCryptoBlake2s128 = RustCryptoBlake2s; fn oneshot(c: &mut Criterion) { use dryoc::classic::crypto_generichash::crypto_generichash; @@ -38,7 +42,8 @@ fn oneshot(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("dryoc/blake2b256", len), data, |b, d| { let mut out = [0u8; 32]; b.iter(|| { - crypto_generichash(black_box(&mut out), black_box(d), None).unwrap(); + crypto_generichash(black_box(&mut out), black_box(d), None) + .expect("valid BLAKE2 benchmark operation must succeed"); black_box(out) }) }); @@ -52,7 +57,8 @@ fn oneshot(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("dryoc/blake2b512", len), data, |b, d| { let mut out = [0u8; 64]; b.iter(|| { - crypto_generichash(black_box(&mut out), black_box(d), None).unwrap(); + crypto_generichash(black_box(&mut out), black_box(d), None) + .expect("valid BLAKE2 benchmark operation must succeed"); black_box(out) }) }); @@ -86,8 +92,8 @@ fn host_overhead(c: &mut Criterion) { let inputs = tiny_inputs(); let key_b = [0x42u8; 64]; let key_s = [0x24u8; 32]; - let key_b_typed = Blake2bKey::new(black_box(&key_b[..32])).unwrap(); - let key_s_typed = Blake2sKey::new(black_box(&key_s)).unwrap(); + let key_b_typed = Blake2bKey::new(black_box(&key_b[..32])).expect("valid BLAKE2 benchmark operation must succeed"); + let key_s_typed = Blake2sKey::new(black_box(&key_s)).expect("valid BLAKE2 benchmark operation must succeed"); let mut oneshot = c.benchmark_group("blake2/host-overhead"); for (len, data) in &inputs { @@ -118,7 +124,8 @@ fn host_overhead(c: &mut Criterion) { }); keyed.bench_with_input(BenchmarkId::new("rustcrypto/blake2b256", len), data, |b, d| { b.iter(|| { - let mut mac = RustCryptoBlake2bMac256::new_from_slice(black_box(&key_b[..32])).unwrap(); + let mut mac = RustCryptoBlake2bMac256::new_from_slice(black_box(&key_b[..32])) + .expect("valid BLAKE2 benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -129,7 +136,8 @@ fn host_overhead(c: &mut Criterion) { }); keyed.bench_with_input(BenchmarkId::new("rustcrypto/blake2s256", len), data, |b, d| { b.iter(|| { - let mut mac = RustCryptoBlake2sMac256::new_from_slice(black_box(&key_s)).unwrap(); + let mut mac = RustCryptoBlake2sMac256::new_from_slice(black_box(&key_s)) + .expect("valid BLAKE2 benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -180,10 +188,10 @@ fn keyed(c: &mut Criterion) { let inputs = common::comp_sizes(); let key_b = [0x42u8; 64]; let key_s = [0x24u8; 32]; - let key_b_256 = Blake2bKey::new(black_box(&key_b[..32])).unwrap(); - let key_b_512 = Blake2bKey::new(black_box(&key_b)).unwrap(); - let key_s_128 = Blake2sKey::new(black_box(&key_s[..16])).unwrap(); - let key_s_256 = Blake2sKey::new(black_box(&key_s)).unwrap(); + let key_b_256 = Blake2bKey::new(black_box(&key_b[..32])).expect("valid BLAKE2 benchmark operation must succeed"); + let key_b_512 = Blake2bKey::new(black_box(&key_b)).expect("valid BLAKE2 benchmark operation must succeed"); + let key_s_128 = Blake2sKey::new(black_box(&key_s[..16])).expect("valid BLAKE2 benchmark operation must succeed"); + let key_s_256 = Blake2sKey::new(black_box(&key_s)).expect("valid BLAKE2 benchmark operation must succeed"); let mut g = c.benchmark_group("blake2/keyed"); for (len, data) in &inputs { @@ -194,7 +202,8 @@ fn keyed(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto/blake2b256", len), data, |b, d| { b.iter(|| { - let mut mac = RustCryptoBlake2bMac256::new_from_slice(black_box(&key_b[..32])).unwrap(); + let mut mac = RustCryptoBlake2bMac256::new_from_slice(black_box(&key_b[..32])) + .expect("valid BLAKE2 benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -202,7 +211,8 @@ fn keyed(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("dryoc/blake2b256", len), data, |b, d| { let mut out = [0u8; 32]; b.iter(|| { - crypto_generichash(black_box(&mut out), black_box(d), Some(black_box(&key_b[..32]))).unwrap(); + crypto_generichash(black_box(&mut out), black_box(d), Some(black_box(&key_b[..32]))) + .expect("valid BLAKE2 benchmark operation must succeed"); black_box(out) }) }); @@ -212,7 +222,8 @@ fn keyed(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto/blake2b512", len), data, |b, d| { b.iter(|| { - let mut mac = RustCryptoBlake2bMac512::new_from_slice(black_box(&key_b)).unwrap(); + let mut mac = RustCryptoBlake2bMac512::new_from_slice(black_box(&key_b)) + .expect("valid BLAKE2 benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -220,7 +231,8 @@ fn keyed(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("dryoc/blake2b512", len), data, |b, d| { let mut out = [0u8; 64]; b.iter(|| { - crypto_generichash(black_box(&mut out), black_box(d), Some(black_box(&key_b[..]))).unwrap(); + crypto_generichash(black_box(&mut out), black_box(d), Some(black_box(&key_b[..]))) + .expect("valid BLAKE2 benchmark operation must succeed"); black_box(out) }) }); @@ -230,7 +242,8 @@ fn keyed(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto/blake2s128", len), data, |b, d| { b.iter(|| { - let mut mac = RustCryptoBlake2sMac128::new_from_slice(black_box(&key_s[..16])).unwrap(); + let mut mac = RustCryptoBlake2sMac128::new_from_slice(black_box(&key_s[..16])) + .expect("valid BLAKE2 benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -241,7 +254,8 @@ fn keyed(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto/blake2s256", len), data, |b, d| { b.iter(|| { - let mut mac = RustCryptoBlake2sMac256::new_from_slice(black_box(&key_s)).unwrap(); + let mut mac = RustCryptoBlake2sMac256::new_from_slice(black_box(&key_s)) + .expect("valid BLAKE2 benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -281,12 +295,12 @@ fn streaming(c: &mut Criterion) { }); g.bench_function(format!("dryoc/blake2b256/{chunk_size}B"), |b| { b.iter(|| { - let mut state = crypto_generichash_init(None, 32).unwrap(); + let mut state = crypto_generichash_init(None, 32).expect("valid BLAKE2 benchmark operation must succeed"); for chunk in data.chunks(chunk_size) { crypto_generichash_update(&mut state, black_box(chunk)); } let mut out = [0u8; 32]; - crypto_generichash_final(state, &mut out).unwrap(); + crypto_generichash_final(state, &mut out).expect("valid BLAKE2 benchmark operation must succeed"); black_box(out) }) }); diff --git a/benches/blake3.rs b/benches/blake3.rs index 34d87f48..0cc81a25 100644 --- a/benches/blake3.rs +++ b/benches/blake3.rs @@ -132,7 +132,14 @@ fn oneshot(c: &mut Criterion) { g.bench_with_input( BenchmarkId::new(format!("rscrypto-{}", kernel.label()), len), data, - |b, d| b.iter(|| black_box(diag_blake3_digest_with_kernel(kernel, black_box(d)).unwrap())), + |b, d| { + b.iter(|| { + black_box( + diag_blake3_digest_with_kernel(kernel, black_box(d)) + .expect("selected BLAKE3 benchmark kernel must be available"), + ) + }) + }, ); } @@ -167,7 +174,12 @@ fn keyed(c: &mut Criterion) { BenchmarkId::new(format!("rscrypto-{}", kernel.label()), len), data, |b, d| { - b.iter(|| black_box(diag_blake3_keyed_digest_with_kernel(kernel, black_box(&key), black_box(d)).unwrap())) + b.iter(|| { + black_box( + diag_blake3_keyed_digest_with_kernel(kernel, black_box(&key), black_box(d)) + .expect("selected BLAKE3 benchmark kernel must be available"), + ) + }) }, ); } @@ -228,7 +240,12 @@ fn streaming(c: &mut Criterion) { continue; } g.bench_function(format!("rscrypto-{}/{chunk_size}B", kernel.label()), |b| { - b.iter(|| black_box(diag_blake3_streaming_digest_with_kernel(kernel, black_box(&data), chunk_size).unwrap())) + b.iter(|| { + black_box( + diag_blake3_streaming_digest_with_kernel(kernel, black_box(&data), chunk_size) + .expect("selected BLAKE3 benchmark kernel must be available"), + ) + }) }); } @@ -279,7 +296,8 @@ fn xof(c: &mut Criterion) { |b, d| { b.iter(|| { let mut out = [0u8; OUT_LEN]; - diag_blake3_xof_with_kernel(kernel, black_box(d), &mut out).unwrap(); + diag_blake3_xof_with_kernel(kernel, black_box(d), &mut out) + .expect("selected BLAKE3 benchmark kernel must be available"); black_box(out) }) }, @@ -335,7 +353,8 @@ fn xof_output(c: &mut Criterion) { |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - diag_blake3_xof_with_kernel(kernel, black_box(&data), black_box(out.as_mut_slice())).unwrap(); + diag_blake3_xof_with_kernel(kernel, black_box(&data), black_box(out.as_mut_slice())) + .expect("selected BLAKE3 benchmark kernel must be available"); black_box(out[0]) }) }, @@ -368,7 +387,7 @@ fn tail_diagnostics(c: &mut Criterion) { let mut digest_group = c.benchmark_group("blake3/chunk-tail-digest"); for chunks in tail_counts.iter().copied() { - let data = common::random_bytes(chunks * BLAKE3_CHUNK_LEN); + let data = common::random_bytes(chunks.strict_mul(BLAKE3_CHUNK_LEN)); common::set_throughput(&mut digest_group, data.len()); digest_group.bench_with_input(BenchmarkId::new("rscrypto", chunks), &data, |b, d| { @@ -382,7 +401,14 @@ fn tail_diagnostics(c: &mut Criterion) { digest_group.bench_with_input( BenchmarkId::new(format!("rscrypto-{}", kernel.label()), chunks), &data, - |b, d| b.iter(|| black_box(diag_blake3_digest_with_kernel(kernel, black_box(d)).unwrap())), + |b, d| { + b.iter(|| { + black_box( + diag_blake3_digest_with_kernel(kernel, black_box(d)) + .expect("selected BLAKE3 benchmark kernel must be available"), + ) + }) + }, ); } @@ -394,20 +420,21 @@ fn tail_diagnostics(c: &mut Criterion) { let mut chunk_group = c.benchmark_group("blake3/chunk-tail-cvs"); for chunks in tail_counts.iter().copied() { - let data = common::random_bytes(chunks * BLAKE3_CHUNK_LEN); + let data = common::random_bytes(chunks.strict_mul(BLAKE3_CHUNK_LEN)); common::set_throughput(&mut chunk_group, data.len()); for &kernel in chunk_tail_diag_kernels() { if !diag_blake3_kernel_available(kernel) { continue; } - let mut out = vec![0u8; chunks * BLAKE3_OUT_LEN]; + let mut out = vec![0u8; chunks.strict_mul(BLAKE3_OUT_LEN)]; chunk_group.bench_with_input( BenchmarkId::new(format!("rscrypto-{}", kernel.label()), chunks), &data, |b, d| { b.iter(|| { - diag_blake3_chunk_cvs_with_kernel(kernel, black_box(d), black_box(out.as_mut_slice())).unwrap(); + diag_blake3_chunk_cvs_with_kernel(kernel, black_box(d), black_box(out.as_mut_slice())) + .expect("selected BLAKE3 benchmark kernel must be available"); black_box(out[0]) }) }, @@ -418,20 +445,21 @@ fn tail_diagnostics(c: &mut Criterion) { let mut parent_group = c.benchmark_group("blake3/parent-tail-cvs"); for parents in tail_counts { - let children = common::random_bytes(parents * 2 * BLAKE3_OUT_LEN); + let children = common::random_bytes(parents.strict_mul(2).strict_mul(BLAKE3_OUT_LEN)); common::set_throughput(&mut parent_group, children.len()); for &kernel in parent_tail_diag_kernels() { if !diag_blake3_kernel_available(kernel) { continue; } - let mut out = vec![0u8; parents * BLAKE3_OUT_LEN]; + let mut out = vec![0u8; parents.strict_mul(BLAKE3_OUT_LEN)]; parent_group.bench_with_input( BenchmarkId::new(format!("rscrypto-{}", kernel.label()), parents), &children, |b, d| { b.iter(|| { - diag_blake3_parent_cvs_with_kernel(kernel, black_box(d), black_box(out.as_mut_slice())).unwrap(); + diag_blake3_parent_cvs_with_kernel(kernel, black_box(d), black_box(out.as_mut_slice())) + .expect("selected BLAKE3 benchmark kernel must be available"); black_box(out[0]) }) }, diff --git a/benches/common/mod.rs b/benches/common/mod.rs index 1acda7aa..2cb1011e 100644 --- a/benches/common/mod.rs +++ b/benches/common/mod.rs @@ -1,7 +1,7 @@ use criterion::{BenchmarkGroup, Throughput, measurement::WallTime}; /// Deterministic pseudo-random bytes for reproducible benchmarks. -pub fn random_bytes(len: usize) -> Vec { +pub(crate) fn random_bytes(len: usize) -> Vec { let mut state: u64 = (len as u64) ^ 0x517c_c1b7_2722_0a95; (0..len) .map(|_| { @@ -19,7 +19,7 @@ pub fn random_bytes(len: usize) -> Vec { /// Covers: overhead (0B, 1B), small (32B), block boundary (64B), /// medium (256B, 1 KiB), page-aligned (4 KiB, 16 KiB), in-cache /// (64 KiB, 256 KiB), throughput (1 MiB). -pub fn comp_sizes() -> Vec<(usize, Vec)> { +pub(crate) fn comp_sizes() -> Vec<(usize, Vec)> { [0, 1, 32, 64, 256, 1024, 4096, 16384, 65536, 262144, 1048576] .into_iter() .map(|len| (len, random_bytes(len))) @@ -27,7 +27,7 @@ pub fn comp_sizes() -> Vec<(usize, Vec)> { } /// Set criterion throughput for a benchmark group. -pub fn set_throughput(group: &mut BenchmarkGroup<'_, WallTime>, len: usize) { +pub(crate) fn set_throughput(group: &mut BenchmarkGroup<'_, WallTime>, len: usize) { if len > 0 { group.throughput(Throughput::Bytes(len as u64)); } diff --git a/benches/password_hashing.rs b/benches/password_hashing.rs index c6037893..8b7ea8e2 100644 --- a/benches/password_hashing.rs +++ b/benches/password_hashing.rs @@ -4,12 +4,13 @@ //! Organised by cost-parameter classes so CI can run the fast groups on //! every push and reserve the OWASP-scale group for dedicated perf runs. -#![allow(clippy::unwrap_used)] - use core::{hint::black_box, time::Duration}; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; -use dryoc::classic::crypto_pwhash::{PasswordHashAlgorithm, crypto_pwhash}; +use dryoc::{ + classic::crypto_pwhash::{PasswordHashAlgorithm, crypto_pwhash}, + constants::{CRYPTO_PWHASH_ARGON2I_OPSLIMIT_MIN, CRYPTO_PWHASH_ARGON2ID_OPSLIMIT_MIN}, +}; use rscrypto::{ Argon2Error, Argon2Params, Argon2d, Argon2i, Argon2id, Argon2idPassword, Scrypt, ScryptParams, ScryptPassword, }; @@ -22,12 +23,13 @@ const SALT: &[u8] = b"rscrypto-bench-salt-16bytes!"; /// Build rscrypto params. fn rs_params(m_kib: u32, t: u32, p: u32, _out_len: u32) -> Argon2Params { - Argon2Params::new(m_kib, t, p).unwrap() + Argon2Params::new(m_kib, t, p).expect("supported password-hashing benchmark parameters must succeed") } /// Build RustCrypto oracle context. fn oracle_ctx(algo: argon2::Algorithm, m_kib: u32, t: u32, p: u32, out_len: usize) -> argon2::Argon2<'static> { - let params = argon2::Params::new(m_kib, t, p, Some(out_len)).unwrap(); + let params = argon2::Params::new(m_kib, t, p, Some(out_len)) + .expect("supported password-hashing benchmark parameters must succeed"); argon2::Argon2::new(algo, argon2::Version::V0x13, params) } @@ -40,6 +42,15 @@ const SMALL_MATRIX: &[(u32, u32, u32)] = &[ (64, 3, 2), ]; +fn dryoc_supports_small_row(algorithm: PasswordHashAlgorithm, time_cost: u32, parallelism: u32) -> bool { + let minimum_time_cost = match algorithm { + PasswordHashAlgorithm::Argon2i13 => CRYPTO_PWHASH_ARGON2I_OPSLIMIT_MIN, + PasswordHashAlgorithm::Argon2id13 => CRYPTO_PWHASH_ARGON2ID_OPSLIMIT_MIN, + }; + + parallelism == 1 && u64::from(time_cost) >= minimum_time_cost +} + fn bench_small_variant( c: &mut Criterion, group_name: &str, @@ -53,7 +64,12 @@ fn bench_small_variant( for &(m, t, p) in SMALL_MATRIX { let out_len = 32usize; let param_id = format!("m={m}_t={t}_p={p}"); - let rs_params = rs_params(m, t, p, out_len as u32); + let rs_params = rs_params( + m, + t, + p, + u32::try_from(out_len).expect("benchmark output length must fit u32"), + ); let oracle = oracle_ctx(oracle_algo, m, t, p, out_len); g.bench_with_input(BenchmarkId::new("rscrypto", ¶m_id), &rs_params, |b, params| { @@ -65,7 +81,7 @@ fn bench_small_variant( black_box(SALT), black_box(&mut out), ) - .unwrap() + .expect("supported password-hashing benchmark parameters must succeed") }); }); @@ -74,28 +90,26 @@ fn bench_small_variant( b.iter(|| { ctx .hash_password_into(black_box(PASSWORD), black_box(SALT), black_box(&mut out)) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); - // dryoc wraps libsodium's `crypto_pwhash`, which fixes parallelism at p=1. - // Skip the row whenever the matrix asks for p>1; that case is rscrypto+rustcrypto only. - if p == 1 - && let Some(ref alg) = dryoc_algo - { - let memlimit_bytes = (m as usize).saturating_mul(1024); - g.bench_with_input(BenchmarkId::new("dryoc", ¶m_id), alg, |b, algorithm| { + if let Some(algorithm) = dryoc_algo.filter(|&algorithm| dryoc_supports_small_row(algorithm, t, p)) { + let memlimit_bytes = usize::try_from(m) + .expect("benchmark memory cost must fit usize") + .strict_mul(1024); + g.bench_with_input(BenchmarkId::new("dryoc", ¶m_id), &algorithm, |b, algorithm| { let mut out = [0u8; 32]; b.iter(|| { crypto_pwhash( black_box(&mut out), black_box(PASSWORD), black_box(&SALT[..16]), - t as u64, + u64::from(t), memlimit_bytes, - algorithm.clone(), + *algorithm, ) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); } @@ -139,7 +153,12 @@ fn argon2id_owasp(c: &mut Criterion) { let out_len = 32usize; // OWASP 2024: m=19MiB, t=2, p=1 - let rs_params = rs_params(19 * 1024, 2, 1, out_len as u32); + let rs_params = rs_params( + 19 * 1024, + 2, + 1, + u32::try_from(out_len).expect("benchmark output length must fit u32"), + ); let oracle = oracle_ctx(argon2::Algorithm::Argon2id, 19 * 1024, 2, 1, out_len); g.bench_function(BenchmarkId::new("rscrypto", "m=19MiB_t=2_p=1"), |b| { @@ -151,7 +170,7 @@ fn argon2id_owasp(c: &mut Criterion) { black_box(SALT), black_box(&mut out), ) - .unwrap() + .expect("supported password-hashing benchmark parameters must succeed") }); }); @@ -160,12 +179,12 @@ fn argon2id_owasp(c: &mut Criterion) { b.iter(|| { oracle .hash_password_into(black_box(PASSWORD), black_box(SALT), black_box(&mut out)) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); // dryoc / libsodium-classic Argon2id at OWASP parameters (memlimit in bytes). - let dryoc_memlimit = (19 * 1024usize).saturating_mul(1024); + let dryoc_memlimit = 19usize.strict_mul(1024).strict_mul(1024); g.bench_function(BenchmarkId::new("dryoc", "m=19MiB_t=2_p=1"), |b| { let mut out = [0u8; 32]; b.iter(|| { @@ -177,7 +196,7 @@ fn argon2id_owasp(c: &mut Criterion) { dryoc_memlimit, PasswordHashAlgorithm::Argon2id13, ) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); @@ -186,12 +205,12 @@ fn argon2id_owasp(c: &mut Criterion) { /// Build rscrypto scrypt params. fn rs_scrypt_params(log_n: u8, r: u32, p: u32, _out_len: u32) -> ScryptParams { - ScryptParams::new(log_n, r, p).unwrap() + ScryptParams::new(log_n, r, p).expect("supported password-hashing benchmark parameters must succeed") } /// Build RustCrypto scrypt oracle params. fn oracle_scrypt_params(log_n: u8, r: u32, p: u32, _out_len: usize) -> scrypt::Params { - scrypt::Params::new(log_n, r, p).unwrap() + scrypt::Params::new(log_n, r, p).expect("supported password-hashing benchmark parameters must succeed") } /// Small / CI-friendly scrypt matrix: (log_n, r, p). @@ -204,7 +223,12 @@ fn scrypt_small(c: &mut Criterion) { for &(log_n, r, p) in SCRYPT_SMALL_MATRIX { let out_len = 32usize; let id = format!("log_n={log_n}_r={r}_p={p}"); - let rs = rs_scrypt_params(log_n, r, p, out_len as u32); + let rs = rs_scrypt_params( + log_n, + r, + p, + u32::try_from(out_len).expect("benchmark output length must fit u32"), + ); let oracle = oracle_scrypt_params(log_n, r, p, out_len); g.bench_with_input(BenchmarkId::new("rscrypto", &id), &rs, |b, params| { @@ -216,14 +240,15 @@ fn scrypt_small(c: &mut Criterion) { black_box(SALT), black_box(&mut out), ) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); g.bench_with_input(BenchmarkId::new("rustcrypto", &id), &oracle, |b, params| { let mut out = [0u8; 32]; b.iter(|| { - scrypt::scrypt(black_box(PASSWORD), black_box(SALT), params, black_box(&mut out)).unwrap(); + scrypt::scrypt(black_box(PASSWORD), black_box(SALT), params, black_box(&mut out)) + .expect("supported password-hashing benchmark parameters must succeed"); }); }); } @@ -238,7 +263,12 @@ fn scrypt_owasp(c: &mut Criterion) { g.measurement_time(Duration::from_secs(30)); let out_len = 32usize; - let rs = rs_scrypt_params(17, 8, 1, out_len as u32); + let rs = rs_scrypt_params( + 17, + 8, + 1, + u32::try_from(out_len).expect("benchmark output length must fit u32"), + ); let oracle = oracle_scrypt_params(17, 8, 1, out_len); g.bench_function(BenchmarkId::new("rscrypto", "log_n=17_r=8_p=1"), |b| { @@ -250,14 +280,15 @@ fn scrypt_owasp(c: &mut Criterion) { black_box(SALT), black_box(&mut out), ) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); g.bench_function(BenchmarkId::new("rustcrypto", "log_n=17_r=8_p=1"), |b| { let mut out = [0u8; 32]; b.iter(|| { - scrypt::scrypt(black_box(PASSWORD), black_box(SALT), &oracle, black_box(&mut out)).unwrap(); + scrypt::scrypt(black_box(PASSWORD), black_box(SALT), &oracle, black_box(&mut out)) + .expect("supported password-hashing benchmark parameters must succeed"); }); }); @@ -270,17 +301,23 @@ fn scrypt_phc_roundtrip(c: &mut Criterion) { g.sample_size(20); let params = rs_scrypt_params(10, 8, 1, 32); - let password = ScryptPassword::new(params).unwrap(); + let password = ScryptPassword::new(params).expect("supported password-hashing benchmark parameters must succeed"); g.bench_function("hash_password", |b| { - b.iter(|| password.hash_password(black_box(PASSWORD)).unwrap()); + b.iter(|| { + password + .hash_password(black_box(PASSWORD)) + .expect("supported password-hashing benchmark parameters must succeed") + }); }); - let encoded = password.hash_password(PASSWORD).unwrap(); + let encoded = password + .hash_password(PASSWORD) + .expect("supported password-hashing benchmark parameters must succeed"); g.bench_function("verify_password", |b| { b.iter(|| { password .verify_password(black_box(PASSWORD), black_box(&encoded)) - .unwrap() + .expect("supported password-hashing benchmark parameters must succeed") }); }); @@ -298,17 +335,23 @@ fn argon2id_phc_roundtrip(c: &mut Criterion) { g.sample_size(30); let params = rs_params(32, 2, 1, 32); - let password = Argon2idPassword::new(params).unwrap(); + let password = Argon2idPassword::new(params).expect("supported password-hashing benchmark parameters must succeed"); g.bench_function("hash_password", |b| { - b.iter(|| password.hash_password(black_box(PASSWORD)).unwrap()); + b.iter(|| { + password + .hash_password(black_box(PASSWORD)) + .expect("supported password-hashing benchmark parameters must succeed") + }); }); - let encoded = password.hash_password(PASSWORD).unwrap(); + let encoded = password + .hash_password(PASSWORD) + .expect("supported password-hashing benchmark parameters must succeed"); g.bench_function("verify_password", |b| { b.iter(|| { password .verify_password(black_box(PASSWORD), black_box(&encoded)) - .unwrap() + .expect("supported password-hashing benchmark parameters must succeed") }); }); @@ -336,7 +379,12 @@ fn argon2id_parallel_scaling(c: &mut Criterion) { for &p in &[1u32, 4, 8, 16] { let id = format!("p={p}"); - let params = rs_params(m_kib, t, p, out_len as u32); + let params = rs_params( + m_kib, + t, + p, + u32::try_from(out_len).expect("benchmark output length must fit u32"), + ); g.bench_with_input(BenchmarkId::new("rscrypto", &id), ¶ms, |b, params| { let mut out = [0u8; 32]; @@ -347,7 +395,7 @@ fn argon2id_parallel_scaling(c: &mut Criterion) { black_box(SALT), black_box(&mut out), ) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); } @@ -369,7 +417,12 @@ fn argon2id_parallel_owasp(c: &mut Criterion) { for &p in &[1u32, 4, 8, 16] { let id = format!("p={p}"); - let params = rs_params(m_kib, t, p, out_len as u32); + let params = rs_params( + m_kib, + t, + p, + u32::try_from(out_len).expect("benchmark output length must fit u32"), + ); g.bench_with_input(BenchmarkId::new("rscrypto", &id), ¶ms, |b, params| { let mut out = [0u8; 32]; @@ -380,7 +433,7 @@ fn argon2id_parallel_owasp(c: &mut Criterion) { black_box(SALT), black_box(&mut out), ) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); } diff --git a/benches/rsa.rs b/benches/rsa.rs index 14ec2cd5..831f52b7 100644 --- a/benches/rsa.rs +++ b/benches/rsa.rs @@ -61,38 +61,45 @@ const RSA8192_PKCS1V15_SHA256: &[u8] = include_bytes!("rsa_fixtures/rsa8192_pkcs fn hex_to_vec(hex: &str) -> Vec { let mut out = Vec::with_capacity(hex.len() / 2); for chunk in hex.as_bytes().chunks_exact(2) { - out.push((hex_value(chunk[0]) << 4) | hex_value(chunk[1])); + let high = hex_value(chunk[0]).expect("RSA benchmark fixture must contain hexadecimal digits"); + let low = hex_value(chunk[1]).expect("RSA benchmark fixture must contain hexadecimal digits"); + out.push((high << 4) | low); } out } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("short DER length must fit in one byte")]; } let bytes = len.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .expect("long DER length must contain a non-zero byte"); let len_bytes = &bytes[first_nonzero..]; - let mut out = Vec::with_capacity(1 + len_bytes.len()); - out.push(0x80 | len_bytes.len() as u8); + let mut out = Vec::with_capacity(1usize.strict_add(len_bytes.len())); + out.push(0x80 | u8::try_from(len_bytes.len()).expect("DER length-of-length must fit in one byte")); out.extend_from_slice(len_bytes); out } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + der_len(value.len()).len() + value.len()); + let encoded_len = der_len(value.len()); + let capacity = 1usize.strict_add(encoded_len.len()).strict_add(value.len()); + let mut out = Vec::with_capacity(capacity); out.push(tag); - out.extend_from_slice(&der_len(value.len())); + out.extend_from_slice(&encoded_len); out.extend_from_slice(value); out } @@ -104,7 +111,7 @@ fn sequence(value: &[u8]) -> Vec { fn integer_unsigned(value: &[u8]) -> Vec { let first_nonzero = value.iter().position(|&byte| byte != 0); let value = first_nonzero.map_or(&[0u8][..], |index| &value[index..]); - let mut encoded = Vec::with_capacity(value.len() + usize::from(value[0] & 0x80 != 0)); + let mut encoded = Vec::with_capacity(value.len().strict_add(usize::from(value[0] & 0x80 != 0))); if value[0] & 0x80 != 0 { encoded.push(0); } @@ -114,7 +121,10 @@ fn integer_unsigned(value: &[u8]) -> Vec { fn exponent_bytes(exponent: u64) -> Vec { let bytes = exponent.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap_or(bytes.len() - 1); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .unwrap_or_else(|| bytes.len().strict_sub(1)); bytes[first_nonzero..].to_vec() } @@ -240,9 +250,11 @@ fn rsa_components_for_size( pkcs1_sig: &[u8], import_policy: &RsaPublicKeyPolicy, ) { - let pss_key = RsaPublicKey::from_spki_der_with_policy(pss_spki, import_policy).unwrap(); + let pss_key = + RsaPublicKey::from_spki_der_with_policy(pss_spki, import_policy).expect("valid RSA benchmark fixture must succeed"); let mut pss_scratch = pss_key.public_scratch(); - let pkcs1_key = RsaPublicKey::from_spki_der_with_policy(pkcs1_spki, import_policy).unwrap(); + let pkcs1_key = RsaPublicKey::from_spki_der_with_policy(pkcs1_spki, import_policy) + .expect("valid RSA benchmark fixture must succeed"); let mut pkcs1_scratch = pkcs1_key.public_scratch(); let pss_pkcs1 = pkcs1_der_from_key(&pss_key); let pkcs1_pkcs1 = pkcs1_der_from_key(&pkcs1_key); @@ -254,7 +266,7 @@ fn rsa_components_for_size( let mut pss_encoded = vec![0u8; pss_key.modulus().len()]; pss_key .public_operation_with_scratch(pss_sig, &mut pss_encoded, &mut pss_scratch) - .unwrap(); + .expect("valid RSA benchmark fixture must succeed"); let pss_em_bits = pss_key.modulus_bits().strict_sub(1); let pss_em_len = pss_em_bits.strict_add(7) / 8; let leading = pss_encoded.len().strict_sub(pss_em_len); @@ -263,7 +275,7 @@ fn rsa_components_for_size( let mut pkcs1_encoded = vec![0u8; pkcs1_key.modulus().len()]; pkcs1_key .public_operation_with_scratch(pkcs1_sig, &mut pkcs1_encoded, &mut pkcs1_scratch) - .unwrap(); + .expect("valid RSA benchmark fixture must succeed"); ( pss_encoded, @@ -286,20 +298,31 @@ fn rsa_components_for_size( let rustcrypto_pss_key = RustCryptoRsaPublicKey::from_public_key_der(pss_spki) .ok() .map(RustCryptoPssVerifyingKey::::new); - let rustcrypto_pss_sig = RustCryptoPssSignature::try_from(pss_sig).unwrap(); + let rustcrypto_pss_sig = RustCryptoPssSignature::try_from(pss_sig).expect("valid RSA benchmark fixture must succeed"); let rustcrypto_pkcs1_key = RustCryptoRsaPublicKey::from_public_key_der(pkcs1_spki) .ok() .map(RustCryptoPkcs1v15VerifyingKey::::new); - let rustcrypto_pkcs1_sig = RustCryptoPkcs1v15Signature::try_from(pkcs1_sig).unwrap(); + let rustcrypto_pkcs1_sig = + RustCryptoPkcs1v15Signature::try_from(pkcs1_sig).expect("valid RSA benchmark fixture must succeed"); let mut group = c.benchmark_group(name); group.bench_function("parse-spki-rscrypto", |b| { - b.iter(|| black_box(RsaPublicKey::from_spki_der_with_policy(black_box(pss_spki), import_policy).unwrap())) + b.iter(|| { + black_box( + RsaPublicKey::from_spki_der_with_policy(black_box(pss_spki), import_policy) + .expect("valid RSA benchmark fixture must succeed"), + ) + }) }); if rustcrypto_pss_key.is_some() { group.bench_function("parse-spki-rustcrypto-rsa", |b| { - b.iter(|| black_box(RustCryptoRsaPublicKey::from_public_key_der(black_box(pss_spki)).unwrap())) + b.iter(|| { + black_box( + RustCryptoRsaPublicKey::from_public_key_der(black_box(pss_spki)) + .expect("valid RSA benchmark fixture must succeed"), + ) + }) }); } group.bench_function("scratch-setup-rscrypto", |b| { @@ -313,7 +336,7 @@ fn rsa_components_for_size( black_box(&mut out), black_box(&mut pss_scratch), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -328,7 +351,7 @@ fn rsa_components_for_size( black_box(&mut out), black_box(&mut product_scratch), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("public-op-e65537-cios-candidate", |b| { @@ -339,7 +362,7 @@ fn rsa_components_for_size( black_box(&mut out), black_box(&mut cios_scratch), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); } @@ -354,7 +377,7 @@ fn rsa_components_for_size( black_box(&mut pss_db), black_box(&mut pss_db_mask), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -365,7 +388,7 @@ fn rsa_components_for_size( black_box(MESSAGE_PKCS1V15), black_box(&pkcs1_encoded), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("verify-pss-sha256-rscrypto", |b| { @@ -377,22 +400,22 @@ fn rsa_components_for_size( black_box(pss_sig), black_box(&mut pss_scratch), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("verify-pss-sha256-rscrypto-oneshot", |b| { b.iter(|| { pss_key .verify_pss(RsaPssProfile::Sha256, black_box(MESSAGE_PSS), black_box(pss_sig)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("verify-pss-sha256-rscrypto-cold", |b| { b.iter(|| { RsaPublicKey::from_spki_der_with_policy(black_box(pss_spki), import_policy) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") .verify_pss(RsaPssProfile::Sha256, black_box(MESSAGE_PSS), black_box(pss_sig)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); if let Some(rustcrypto_pss_key) = rustcrypto_pss_key { @@ -400,16 +423,20 @@ fn rsa_components_for_size( b.iter(|| { rustcrypto_pss_key .verify(black_box(MESSAGE_PSS), black_box(&rustcrypto_pss_sig)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); } group.bench_function("verify-pss-sha256-ring", |b| { - b.iter(|| ring_pss_key.verify(black_box(MESSAGE_PSS), black_box(pss_sig)).unwrap()) + b.iter(|| { + ring_pss_key + .verify(black_box(MESSAGE_PSS), black_box(pss_sig)) + .expect("valid RSA benchmark fixture must succeed") + }) }); aws_lc_bench! { group.bench_function("verify-pss-sha256-aws-lc-rs", |b| { - b.iter(|| aws_pss_key.verify(black_box(MESSAGE_PSS), black_box(pss_sig)).unwrap()) + b.iter(|| aws_pss_key.verify(black_box(MESSAGE_PSS), black_box(pss_sig)).expect("valid RSA benchmark fixture must succeed")) }); } group.bench_function("verify-pkcs1v15-sha256-rscrypto", |b| { @@ -421,7 +448,7 @@ fn rsa_components_for_size( black_box(pkcs1_sig), black_box(&mut pkcs1_scratch), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("verify-pkcs1v15-sha256-rscrypto-oneshot", |b| { @@ -432,19 +459,19 @@ fn rsa_components_for_size( black_box(MESSAGE_PKCS1V15), black_box(pkcs1_sig), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("verify-pkcs1v15-sha256-rscrypto-cold", |b| { b.iter(|| { RsaPublicKey::from_spki_der_with_policy(black_box(pkcs1_spki), import_policy) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") .verify_pkcs1v15( RsaPkcs1v15Profile::Sha256, black_box(MESSAGE_PKCS1V15), black_box(pkcs1_sig), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); if let Some(rustcrypto_pkcs1_key) = rustcrypto_pkcs1_key { @@ -452,7 +479,7 @@ fn rsa_components_for_size( b.iter(|| { rustcrypto_pkcs1_key .verify(black_box(MESSAGE_PKCS1V15), black_box(&rustcrypto_pkcs1_sig)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); } @@ -460,7 +487,7 @@ fn rsa_components_for_size( b.iter(|| { ring_pkcs1_key .verify(black_box(MESSAGE_PKCS1V15), black_box(pkcs1_sig)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); aws_lc_bench! { @@ -468,7 +495,7 @@ fn rsa_components_for_size( b.iter(|| { aws_pkcs1_key .verify(black_box(MESSAGE_PKCS1V15), black_box(pkcs1_sig)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); } @@ -481,18 +508,18 @@ fn rsa_public_exponents(c: &mut Criterion) { let input = legacy_exponent_input(); let policy = RsaPublicKeyPolicy::legacy_verification().allow_legacy_small_exponents(); - let key_e3 = - RsaPublicKey::from_pkcs1_der_with_policy(&pkcs1_der_from_modulus_exponent(&modulus, &[0x03]), &policy).unwrap(); - let key_e17 = - RsaPublicKey::from_pkcs1_der_with_policy(&pkcs1_der_from_modulus_exponent(&modulus, &[0x11]), &policy).unwrap(); + let key_e3 = RsaPublicKey::from_pkcs1_der_with_policy(&pkcs1_der_from_modulus_exponent(&modulus, &[0x03]), &policy) + .expect("valid RSA benchmark fixture must succeed"); + let key_e17 = RsaPublicKey::from_pkcs1_der_with_policy(&pkcs1_der_from_modulus_exponent(&modulus, &[0x11]), &policy) + .expect("valid RSA benchmark fixture must succeed"); let key_e65537 = RsaPublicKey::from_pkcs1_der_with_policy(&pkcs1_der_from_modulus_exponent(&modulus, &[0x01, 0x00, 0x01]), &policy) - .unwrap(); + .expect("valid RSA benchmark fixture must succeed"); let key_generic = RsaPublicKey::from_pkcs1_der_with_policy( &pkcs1_der_from_modulus_exponent(&modulus, &[0x49, 0xd2, 0xa1]), &policy.allow_legacy_odd_exponents(), ) - .unwrap(); + .expect("valid RSA benchmark fixture must succeed"); let mut scratch_e3 = key_e3.public_scratch(); let mut scratch_e17 = key_e17.public_scratch(); @@ -527,12 +554,15 @@ fn rsa_public_exponents(c: &mut Criterion) { b.iter(|| { key_e3 .public_operation_with_scratch(black_box(&input), black_box(&mut out), black_box(&mut scratch_e3)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] group.bench_function("public-op-e3-bitserial-baseline", |b| { - b.iter(|| diag_rsa_public_operation_bitserial(black_box(&key_e3), black_box(&input), black_box(&mut out)).unwrap()) + b.iter(|| { + diag_rsa_public_operation_bitserial(black_box(&key_e3), black_box(&input), black_box(&mut out)) + .expect("valid RSA benchmark fixture must succeed") + }) }); #[cfg(feature = "diag")] group.bench_function("public-op-e3-product-montgomery", |b| { @@ -543,7 +573,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut product_scratch_e3), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -555,7 +585,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut generic_scratch_e3), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -567,19 +597,22 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut cios_scratch_e3), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("public-op-e17", |b| { b.iter(|| { key_e17 .public_operation_with_scratch(black_box(&input), black_box(&mut out), black_box(&mut scratch_e17)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] group.bench_function("public-op-e17-bitserial-baseline", |b| { - b.iter(|| diag_rsa_public_operation_bitserial(black_box(&key_e17), black_box(&input), black_box(&mut out)).unwrap()) + b.iter(|| { + diag_rsa_public_operation_bitserial(black_box(&key_e17), black_box(&input), black_box(&mut out)) + .expect("valid RSA benchmark fixture must succeed") + }) }); #[cfg(feature = "diag")] group.bench_function("public-op-e17-product-montgomery", |b| { @@ -590,7 +623,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut product_scratch_e17), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -602,7 +635,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut generic_scratch_e17), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -614,20 +647,21 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut cios_scratch_e17), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("public-op-e65537", |b| { b.iter(|| { key_e65537 .public_operation_with_scratch(black_box(&input), black_box(&mut out), black_box(&mut scratch_e65537)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] group.bench_function("public-op-e65537-bitserial-baseline", |b| { b.iter(|| { - diag_rsa_public_operation_bitserial(black_box(&key_e65537), black_box(&input), black_box(&mut out)).unwrap() + diag_rsa_public_operation_bitserial(black_box(&key_e65537), black_box(&input), black_box(&mut out)) + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -639,7 +673,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut product_scratch_e65537), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -651,7 +685,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut generic_scratch_e65537), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -663,20 +697,21 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut cios_scratch_e65537), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("public-op-e0x49d2a1-generic", |b| { b.iter(|| { key_generic .public_operation_with_scratch(black_box(&input), black_box(&mut out), black_box(&mut scratch_generic)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] group.bench_function("public-op-e0x49d2a1-bitserial-baseline", |b| { b.iter(|| { - diag_rsa_public_operation_bitserial(black_box(&key_generic), black_box(&input), black_box(&mut out)).unwrap() + diag_rsa_public_operation_bitserial(black_box(&key_generic), black_box(&input), black_box(&mut out)) + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -688,7 +723,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut product_scratch_generic), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -700,7 +735,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut cios_scratch_generic), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.finish(); @@ -748,7 +783,7 @@ fn rsa_montgomery_thresholds(c: &mut Criterion) { &synthetic_pkcs1_der(modulus_len), &RsaPublicKeyPolicy::legacy_verification(), ) - .unwrap(); + .expect("valid RSA benchmark fixture must succeed"); let input = modulus_minus_one(&key); let mut out_auto = vec![0u8; key.modulus().len()]; let mut out_product = vec![0u8; key.modulus().len()]; @@ -767,7 +802,7 @@ fn rsa_montgomery_thresholds(c: &mut Criterion) { black_box(&mut out_auto), black_box(&mut scratch_auto), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function(format!("{name}/product-montgomery"), |b| { @@ -778,7 +813,7 @@ fn rsa_montgomery_thresholds(c: &mut Criterion) { black_box(&mut out_product), black_box(&mut scratch_product), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function(format!("{name}/cios-candidate"), |b| { @@ -789,7 +824,7 @@ fn rsa_montgomery_thresholds(c: &mut Criterion) { black_box(&mut out_cios), black_box(&mut scratch_cios), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function(format!("{name}/cios-portable"), |b| { @@ -800,7 +835,7 @@ fn rsa_montgomery_thresholds(c: &mut Criterion) { black_box(&mut out_cios_portable), black_box(&mut scratch_cios_portable), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); } diff --git a/benches/sha2.rs b/benches/sha2.rs index 293da66c..3205c8f4 100644 --- a/benches/sha2.rs +++ b/benches/sha2.rs @@ -200,7 +200,7 @@ fn sha256_internal(c: &mut Criterion) { let mut g = c.benchmark_group("sha256/internal/compress"); for block_count in [1usize, 2, 16] { - let len = block_count * 64; + let len = block_count.strict_mul(64); common::set_throughput(&mut g, len); g.bench_with_input(BenchmarkId::new("selected-kernel", format!("{len}B")), &len, |b, &n| { let blocks = &blocks[..n]; diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 00000000..be54ebfb --- /dev/null +++ b/clippy.toml @@ -0,0 +1,2 @@ +check-private-items = true +check-incompatible-msrv-in-tests = true diff --git a/deny.toml b/deny.toml index 8b889930..3bfaef86 100644 --- a/deny.toml +++ b/deny.toml @@ -1,25 +1,22 @@ [graph] -# Target coverage for rscrypto - canonical list for dependency analysis -# See .cargo/config.toml for per-target rustflags +# Dependency-analysis projection of .config/target-matrix.json. targets = [ "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", "aarch64-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu", "aarch64-unknown-linux-musl", - "wasm32-unknown-unknown", - "wasm32-wasip1", - "x86_64-unknown-linux-musl", - "aarch64-pc-windows-msvc", "aarch64-unknown-none", - "loongarch64-unknown-linux-gnu", "powerpc64le-unknown-linux-gnu", "riscv32imac-unknown-none-elf", "riscv64gc-unknown-linux-gnu", - "riscv64gc-unknown-none-elf", "s390x-unknown-linux-gnu", "thumbv6m-none-eabi", + "wasm32-unknown-unknown", + "wasm32-wasip1", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", "x86_64-unknown-none", ] diff --git a/docs/features.md b/docs/features.md index 565c2d36..e7b70b36 100644 --- a/docs/features.md +++ b/docs/features.md @@ -14,28 +14,28 @@ algorithm feature explicitly. ```toml # One algorithm, no_std. -rscrypto = { version = "0.7.8", default-features = false, features = ["sha2"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["sha2"] } # RSA public-key import and verification, no_std + alloc. -rscrypto = { version = "0.7.8", default-features = false, features = ["rsa"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["rsa"] } # RSA key generation, signing, encryption, and private-operation blinding. -rscrypto = { version = "0.7.8", default-features = false, features = ["rsa", "getrandom"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["rsa", "getrandom"] } # ECDSA P-256/SHA-256 and P-384/SHA-384 signing and verification. -rscrypto = { version = "0.7.8", default-features = false, features = ["ecdsa"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["ecdsa"] } # FIPS 203 ML-KEM-512/768/1024 KEM APIs with caller-supplied randomness. -rscrypto = { version = "0.7.8", default-features = false, features = ["ml-kem"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["ml-kem"] } # Everything. -rscrypto = { version = "0.7.8", features = ["full", "getrandom"] } +rscrypto = { version = "0.8.1", features = ["full", "getrandom"] } # Everything, with parallel BLAKE3 / Argon2 lanes via Rayon. -rscrypto = { version = "0.7.8", features = ["full", "parallel", "getrandom"] } +rscrypto = { version = "0.8.1", features = ["full", "parallel", "getrandom"] } # Audit-constrained: makes runtime dispatch ignore host SIMD/ASM capabilities. -rscrypto = { version = "0.7.8", features = ["full", "portable-only"] } +rscrypto = { version = "0.8.1", features = ["full", "portable-only"] } ``` ## Complete feature index diff --git a/docs/migration/README.md b/docs/migration/README.md index 01f2ecb3..e5ce58e8 100644 --- a/docs/migration/README.md +++ b/docs/migration/README.md @@ -29,12 +29,12 @@ For projects upgrading rscrypto itself, start with | From | To | Status | |---|---|---| -| [`blake3`](blake3.md) | `Blake3` | Verified against `blake3 1.8.5` | +| [`blake3`](blake3.md) | `Blake3` | Verified against `blake3 1.8.6` | | [`sha2`](RustCrypto/sha2.md) (RustCrypto) | `Sha224`, `Sha256`, `Sha384`, `Sha512`, `Sha512_256` | Verified against `sha2 0.11.0` | | [`sha3`](RustCrypto/sha3.md) (RustCrypto) | `Sha3_224`, `Sha3_256`, `Sha3_384`, `Sha3_512`, `Shake128`, `Shake256`, `Cshake128`, `Cshake256` | Verified against `sha3 0.12.0` | -| [`blake2`](RustCrypto/blake2.md) (RustCrypto) | `Blake2b256`, `Blake2b512`, `Blake2s128`, `Blake2s256` | Verified against `blake2 0.11.0-rc.6` | +| [`blake2`](RustCrypto/blake2.md) (RustCrypto) | `Blake2b256`, `Blake2b512`, `Blake2s128`, `Blake2s256` | Verified against `blake2 0.10.6` | | [`ascon-hash`](RustCrypto/ascon-hash.md) (RustCrypto) | `AsconHash256`, `AsconXof`, `AsconCxof128` | Verified against `ascon-hash 0.4.0` | -| [`xxhash-rust`](xxhash-rust.md) | `Xxh3`, `Xxh3_128`, `Xxh3Hasher`, `Xxh3_128Hasher`, `Xxh3BuildHasher` | Verified against `xxhash-rust 0.8.16` | +| [`xxhash-rust`](xxhash-rust.md) | `Xxh3`, `Xxh3_128`, `Xxh3Hasher`, `Xxh3_128Hasher`, `Xxh3BuildHasher` | Verified against `xxhash-rust 0.8.18` | | [`twox-hash`](twox-hash.md) | `Xxh3`, `Xxh3_128`, `Xxh3Hasher`, `Xxh3_128Hasher`, `Xxh3BuildHasher` | API migration guidance; XXH3 output covered by `xxhash-rust` oracle tests | | [`rapidhash`](rapidhash.md) | `RapidHash64`, `RapidStreamHasher`, `RapidHasher`, `RapidSeededState`, `RapidRandomState` | Verified against `rapidhash 4.5.1` | @@ -53,25 +53,25 @@ For projects upgrading rscrypto itself, start with | From | To | Status | |---|---|---| | [`aes-gcm`](RustCrypto/aes-gcm.md) (RustCrypto) | `Aes128Gcm`, `Aes256Gcm` | Verified against `aes-gcm 0.11.0` | -| [`aes-gcm-siv`](RustCrypto/aes-gcm-siv.md) (RustCrypto) | `Aes128GcmSiv`, `Aes256GcmSiv` | Verified against `aes-gcm-siv 0.11.1` | +| [`aes-gcm-siv`](RustCrypto/aes-gcm-siv.md) (RustCrypto) | `Aes128GcmSiv`, `Aes256GcmSiv` | Verified against `aes-gcm-siv 0.12.0` | | [`chacha20poly1305`](RustCrypto/chacha20poly1305.md) (RustCrypto) | `ChaCha20Poly1305`, `XChaCha20Poly1305` | Verified against `chacha20poly1305 0.11.0` | | [`ascon-aead`](RustCrypto/ascon-aead.md) (RustCrypto) | `AsconAead128` | Verified against `ascon-aead 0.6.0` | -| [`aegis`](aegis.md) | `Aegis256` | Verified against `aegis 0.9.12` | +| [`aegis`](aegis.md) | `Aegis256` | Verified against `aegis 0.9.15` | ## Signatures and key exchange | From | To | Status | |---|---|---| -| [`p256`](RustCrypto/p256.md) / [`p384`](RustCrypto/p384.md) (RustCrypto) | `EcdsaP256SecretKey`, `EcdsaP384SecretKey`, `EcdsaP256PublicKey`, `EcdsaP384PublicKey`, raw/DER signatures | Signing and verification tested against RustCrypto `p256 0.14.0` / `p384 0.13.1` | -| [`ed25519-dalek`](RustCrypto/ed25519-dalek.md) | `Ed25519SecretKey`, `Ed25519PublicKey`, `Ed25519Signature`, `Ed25519Keypair` | Verified against `ed25519-dalek 2.2.0` | +| [`p256`](RustCrypto/p256.md) / [`p384`](RustCrypto/p384.md) (RustCrypto) | `EcdsaP256SecretKey`, `EcdsaP384SecretKey`, `EcdsaP256PublicKey`, `EcdsaP384PublicKey`, raw/DER signatures | Signing and verification tested against RustCrypto `p256 0.14.0` / `p384 0.14.0` | +| [`ed25519-dalek`](RustCrypto/ed25519-dalek.md) | `Ed25519SecretKey`, `Ed25519PublicKey`, `Ed25519Signature`, `Ed25519Keypair` | Verified against `ed25519-dalek 3.0.0` | | [`rsa`](RustCrypto/rsa.md) (RustCrypto) | `RsaPublicKey`, `RsaPrivateKey`, RSA-PSS, RSASSA-PKCS1-v1_5, OAEP | Partial; verified through CAVP, Wycheproof, and RustCrypto/ring/OpenSSL oracles | -| [`x25519-dalek`](RustCrypto/x25519-dalek.md) | `X25519SecretKey`, `X25519PublicKey`, `X25519SharedSecret` | Verified against `x25519-dalek 2.0.1` | +| [`x25519-dalek`](RustCrypto/x25519-dalek.md) | `X25519SecretKey`, `X25519PublicKey`, `X25519SharedSecret` | Verified against `x25519-dalek 3.0.0` | ## Password hashing | From | To | Status | |---|---|---| -| [`argon2`](RustCrypto/argon2.md) (RustCrypto) | Raw `Argon2{d,i,id}` KDFs; bounded `Argon2idPassword` records | Verified against `argon2 0.6.0-rc.8` | +| [`argon2`](RustCrypto/argon2.md) (RustCrypto) | Raw `Argon2{d,i,id}` KDFs; bounded `Argon2idPassword` records | Verified against `argon2 0.5.3` | | [`scrypt`](RustCrypto/scrypt.md) (RustCrypto) | Raw `Scrypt` KDF; bounded `ScryptPassword` records | Verified against `scrypt 0.12.0` | ## Stack migrations diff --git a/docs/migration/RustCrypto/aes-gcm-siv.md b/docs/migration/RustCrypto/aes-gcm-siv.md index b1d666aa..bff269c5 100644 --- a/docs/migration/RustCrypto/aes-gcm-siv.md +++ b/docs/migration/RustCrypto/aes-gcm-siv.md @@ -4,14 +4,14 @@ > `Payload { msg, aad }` with rscrypto's named types and a caller-buffer API. > The RFC 8452 construction and combined ciphertext-and-tag bytes are unchanged. -Verified against `aes-gcm-siv = "0.11.1"` and the `rscrypto` 0.7.8 line. +Verified against `aes-gcm-siv = "0.12.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/aes128gcmsiv_oracle.rs`, `tests/aes256gcmsiv_oracle.rs`, and `tests/aead_wycheproof.rs`. ## TL;DR -| | Before (`aes-gcm-siv` 0.11.x) | After (`rscrypto` 0.7.8) | +| | Before (`aes-gcm-siv` 0.12.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `aes-gcm-siv = "0.11"` | `rscrypto = { version = "0.7.8", features = ["aes-gcm-siv"] }` | +| Cargo dep | `aes-gcm-siv = "0.12"` | `rscrypto = { version = "0.8.1", features = ["aes-gcm-siv"] }` | | Import | `use aes_gcm_siv::{Aes256GcmSiv, Key, Nonce, KeyInit, aead::{Aead, Payload}};` | `use rscrypto::{Aead, Aes256GcmSiv, Aes256GcmSivKey, aead::{Nonce96, expert::AeadWithNonce}};` | | Encrypt | `cipher.encrypt(nonce, Payload { msg, aad })?` | `cipher.encrypt(&nonce, aad, msg, &mut out)?` | @@ -20,13 +20,13 @@ Evidence: `tests/aes128gcmsiv_oracle.rs`, `tests/aes256gcmsiv_oracle.rs`, and `t ```toml # Before [dependencies] -aes-gcm-siv = "0.11" +aes-gcm-siv = "0.12" ``` ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["aes-gcm-siv"] } +rscrypto = { version = "0.8.1", features = ["aes-gcm-siv"] } ``` ## Algorithm map @@ -50,10 +50,10 @@ The migration recipe below uses `Aes256GcmSiv` throughout; substitute use aes_gcm_siv::{Aes256GcmSiv, Key, Nonce, KeyInit}; use aes_gcm_siv::aead::{Aead, Payload}; -let key = Key::::from_slice(&[0u8; 32]); -let cipher = Aes256GcmSiv::new(key); -let nonce = Nonce::from_slice(&[0u8; 12]); -let ct = cipher.encrypt(nonce, Payload { msg: plaintext, aad }).unwrap(); +let key = Key::::from([0u8; 32]); +let cipher = Aes256GcmSiv::new(&key); +let nonce = Nonce::from([0u8; 12]); +let ct = cipher.encrypt(&nonce, Payload { msg: plaintext, aad }).unwrap(); ``` ```rust diff --git a/docs/migration/RustCrypto/aes-gcm.md b/docs/migration/RustCrypto/aes-gcm.md index f0698bae..f21f7304 100644 --- a/docs/migration/RustCrypto/aes-gcm.md +++ b/docs/migration/RustCrypto/aes-gcm.md @@ -4,14 +4,14 @@ Replace the `Aes256Gcm` / `Key` / `Nonce` / `Payload { msg, aad }` builder with rscrypto's named types and a caller-buffer API. The combined ciphertext-and-tag bytes remain interoperable. -Verified against `aes-gcm = "0.11.0"` and the `rscrypto` 0.7.8 line. +Verified against `aes-gcm = "0.11.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/aes128gcm_oracle.rs`, `tests/aes256gcm_oracle.rs`, and `tests/aead_wycheproof.rs`. ## TL;DR -| | Before (`aes-gcm` 0.11.x) | After (`rscrypto` 0.7.8) | +| | Before (`aes-gcm` 0.11.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `aes-gcm = "0.11"` | `rscrypto = { version = "0.7.8", features = ["aes-gcm"] }` | +| Cargo dep | `aes-gcm = "0.11"` | `rscrypto = { version = "0.8.1", features = ["aes-gcm"] }` | | Import | `use aes_gcm::{Aes256Gcm, Key, Nonce, KeyInit, aead::{Aead, Payload}};` | `use rscrypto::{Aead, Aes256Gcm, Aes256GcmKey, aead::{Nonce96, expert::AeadWithNonce}};` | | Encrypt | `cipher.encrypt(nonce, Payload { msg, aad })?` (returns `Vec`) | `cipher.encrypt(&nonce, aad, msg, &mut out)?` (writes into caller buffer) | @@ -26,7 +26,7 @@ aes-gcm = "0.11" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["aes-gcm"] } +rscrypto = { version = "0.8.1", features = ["aes-gcm"] } ``` ## Algorithm map @@ -50,10 +50,10 @@ The migration recipe below uses `Aes256Gcm` throughout; substitute use aes_gcm::{Aes256Gcm, Key, Nonce, KeyInit}; use aes_gcm::aead::{Aead, Payload}; -let key = Key::::from_slice(&[0u8; 32]); -let cipher = Aes256Gcm::new(key); -let nonce = Nonce::from_slice(&[0u8; 12]); -let ct = cipher.encrypt(nonce, Payload { msg: plaintext, aad }).unwrap(); +let key = Key::::from([0u8; 32]); +let cipher = Aes256Gcm::new(&key); +let nonce = Nonce::from([0u8; 12]); +let ct = cipher.encrypt(&nonce, Payload { msg: plaintext, aad }).unwrap(); // ct: Vec, last 16 bytes are the tag. ``` diff --git a/docs/migration/RustCrypto/argon2.md b/docs/migration/RustCrypto/argon2.md index 911ef625..b4f0883b 100644 --- a/docs/migration/RustCrypto/argon2.md +++ b/docs/migration/RustCrypto/argon2.md @@ -5,16 +5,16 @@ rscrypto separates two jobs that should not share an API: - `Argon2d`, `Argon2i`, and `Argon2id` are deterministic raw KDFs. - `Argon2idPassword` generates canonical password records and verifies hostile PHC input under finite resource limits. -The raw implementations are checked against RFC 9106 vectors and the RustCrypto `argon2 0.6.0-rc.8` oracle in `tests/argon2_vectors.rs`, `tests/argon2_differential.rs`, `tests/argon2_kernels.rs`, and `tests/argon2_parallel.rs`. +The raw implementations are checked against RFC 9106 vectors and the RustCrypto `argon2 0.5.3` oracle in `tests/argon2_vectors.rs`, `tests/argon2_differential.rs`, `tests/argon2_kernels.rs`, and `tests/argon2_parallel.rs`. ## Cargo features ```toml # Raw Argon2 KDF -rscrypto = { version = "0.7.8", default-features = false, features = ["argon2"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["argon2"] } # Password-record generation and verification with caller-owned entropy -rscrypto = { version = "0.7.8", default-features = false, features = [ +rscrypto = { version = "0.8.1", default-features = false, features = [ "argon2", "phc-strings", ] } diff --git a/docs/migration/RustCrypto/ascon-aead.md b/docs/migration/RustCrypto/ascon-aead.md index d7cb6374..2c8436e4 100644 --- a/docs/migration/RustCrypto/ascon-aead.md +++ b/docs/migration/RustCrypto/ascon-aead.md @@ -4,14 +4,14 @@ > `Payload { msg, aad }` with rscrypto's named types and a caller-buffer API. > NIST SP 800-232 specifies a 16-byte key, nonce, and tag. -Verified against `ascon-aead = "0.6.0"` and the `rscrypto` 0.7.8 line. +Verified against `ascon-aead = "0.6.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/ascon_aead_oracle.rs`. ## TL;DR -| | Before (`ascon-aead` 0.6.x) | After (`rscrypto` 0.7.8) | +| | Before (`ascon-aead` 0.6.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `ascon-aead = "0.6"` | `rscrypto = { version = "0.7.8", features = ["ascon-aead"] }` | +| Cargo dep | `ascon-aead = "0.6"` | `rscrypto = { version = "0.8.1", features = ["ascon-aead"] }` | | Import | `use ascon_aead::{AsconAead128, Key, Nonce, aead::{Aead, KeyInit, Payload}};` | `use rscrypto::{Aead, AsconAead128, AsconAead128Key, aead::{Nonce128, expert::AeadWithNonce}};` | | Encrypt | `cipher.encrypt(nonce, Payload { msg, aad })?` | `cipher.encrypt(&nonce, aad, msg, &mut out)?` | @@ -26,7 +26,7 @@ ascon-aead = "0.6" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["ascon-aead"] } +rscrypto = { version = "0.8.1", features = ["ascon-aead"] } ``` ## Algorithm map @@ -46,10 +46,10 @@ rscrypto = { version = "0.7.8", features = ["ascon-aead"] } use ascon_aead::{AsconAead128, Key, Nonce}; use ascon_aead::aead::{Aead, KeyInit, Payload}; -let key = Key::::from_slice(&[0u8; 16]); -let cipher = AsconAead128::new(key); -let nonce = Nonce::::from_slice(&[0u8; 16]); -let ct = cipher.encrypt(nonce, Payload { msg: plaintext, aad }).unwrap(); +let key = Key::::from([0u8; 16]); +let cipher = AsconAead128::new(&key); +let nonce = Nonce::::from([0u8; 16]); +let ct = cipher.encrypt(&nonce, Payload { msg: plaintext, aad }).unwrap(); ``` ```rust diff --git a/docs/migration/RustCrypto/ascon-hash.md b/docs/migration/RustCrypto/ascon-hash.md index 51cada0a..538925b3 100644 --- a/docs/migration/RustCrypto/ascon-hash.md +++ b/docs/migration/RustCrypto/ascon-hash.md @@ -4,14 +4,14 @@ > the SP 800-232 Ascon-Hash256 parameter set; the output bytes, `update`, and > `finalize` flow are unchanged. -Verified against `ascon-hash = "0.4.0"` and the `rscrypto` 0.7.8 line. +Verified against `ascon-hash = "0.4.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/ascon_official_vectors.rs`, `tests/ascon_hash_oracle.rs`, `tests/ascon_cxof_vectors.rs`, and `tests/ascon_differential.rs`. ## TL;DR -| | Before (`ascon-hash` 0.4.x) | After (`rscrypto` 0.7.8) | +| | Before (`ascon-hash` 0.4.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `ascon-hash = "0.4"` | `rscrypto = { version = "0.7.8", features = ["ascon-hash"] }` | +| Cargo dep | `ascon-hash = "0.4"` | `rscrypto = { version = "0.8.1", features = ["ascon-hash"] }` | | Import | `use ascon_hash::{AsconHash256, digest::Digest};` | `use rscrypto::{AsconHash256, Digest};` | | Call | `AsconHash256::digest(data)` | `AsconHash256::digest(data)` | @@ -26,7 +26,7 @@ ascon-hash = "0.4" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["ascon-hash"] } +rscrypto = { version = "0.8.1", features = ["ascon-hash"] } ``` ## Algorithm map diff --git a/docs/migration/RustCrypto/blake2.md b/docs/migration/RustCrypto/blake2.md index 740ccb12..907b5a5f 100644 --- a/docs/migration/RustCrypto/blake2.md +++ b/docs/migration/RustCrypto/blake2.md @@ -2,16 +2,16 @@ > Same algorithms (RFC 7693), with named convenience types replacing `Blake2b`-style generic instantiations and `Blake2b256::keyed_digest(key, data)` replacing the separate `Blake2bMac` MAC type. -Verified against `blake2 = "0.11.0-rc.6"` and the `rscrypto` 0.7.8 line. +Verified against `blake2 = "0.10.6"` and the `rscrypto` 0.8.1 line. Evidence: `tests/blake2_official_vectors.rs` and `tests/blake2_differential.rs`. Code samples use the 0.10-style names where they remain the clearest migration shape for existing projects. ## TL;DR -| | Before (`blake2` 0.10.x) | After (`rscrypto` 0.7.8) | +| | Before (`blake2` 0.10.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `blake2 = "0.10"` | `rscrypto = { version = "0.7.8", features = ["blake2b", "blake2s"] }` | +| Cargo dep | `blake2 = "0.10"` | `rscrypto = { version = "0.8.1", features = ["blake2b", "blake2s"] }` | | Import | `use blake2::{Blake2b512, Digest};` | `use rscrypto::{Blake2b512, Digest};` | | Call | `Blake2b512::digest(data)` | `Blake2b512::digest(data)` | @@ -28,7 +28,7 @@ blake2 = "0.10" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["blake2b", "blake2s"] } +rscrypto = { version = "0.8.1", features = ["blake2b", "blake2s"] } ``` ## Algorithm map diff --git a/docs/migration/RustCrypto/chacha20poly1305.md b/docs/migration/RustCrypto/chacha20poly1305.md index 1fb07d1c..e90e6462 100644 --- a/docs/migration/RustCrypto/chacha20poly1305.md +++ b/docs/migration/RustCrypto/chacha20poly1305.md @@ -5,14 +5,14 @@ > ChaCha20-Poly1305 and XChaCha20-Poly1305 operations preserve ciphertext and > tag bytes. -Verified against `chacha20poly1305 = "0.11.0"` and the `rscrypto` 0.7.8 line. +Verified against `chacha20poly1305 = "0.11.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/chacha20poly1305.rs`, `tests/xchacha20poly1305.rs`, and `tests/aead_wycheproof.rs`. ## TL;DR -| | Before (`chacha20poly1305` 0.11.x) | After (`rscrypto` 0.7.8) | +| | Before (`chacha20poly1305` 0.11.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `chacha20poly1305 = "0.11"` | `rscrypto = { version = "0.7.8", features = ["chacha20poly1305", "xchacha20poly1305"] }` | +| Cargo dep | `chacha20poly1305 = "0.11"` | `rscrypto = { version = "0.8.1", features = ["chacha20poly1305", "xchacha20poly1305"] }` | | Import | `use chacha20poly1305::{ChaCha20Poly1305, Key, Nonce, KeyInit, aead::{Aead, Payload}};` | `use rscrypto::{Aead, ChaCha20Poly1305, ChaCha20Poly1305Key, aead::{Nonce96, expert::AeadWithNonce}};` | | Encrypt | `cipher.encrypt(nonce, Payload { msg, aad })?` | `cipher.encrypt(&nonce, aad, msg, &mut out)?` | @@ -29,7 +29,7 @@ chacha20poly1305 = "0.11" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["chacha20poly1305", "xchacha20poly1305"] } +rscrypto = { version = "0.8.1", features = ["chacha20poly1305", "xchacha20poly1305"] } ``` ## Algorithm map @@ -49,10 +49,10 @@ rscrypto = { version = "0.7.8", features = ["chacha20poly1305", "xchacha20poly13 use chacha20poly1305::{ChaCha20Poly1305, Key, Nonce, KeyInit}; use chacha20poly1305::aead::{Aead, Payload}; -let key = Key::from_slice(&[0u8; 32]); // Key is non-generic -let cipher = ChaCha20Poly1305::new(key); -let nonce = Nonce::from_slice(&[0u8; 12]); -let ct = cipher.encrypt(nonce, Payload { msg: plaintext, aad }).unwrap(); +let key = Key::from([0u8; 32]); // Key is non-generic +let cipher = ChaCha20Poly1305::new(&key); +let nonce = Nonce::from([0u8; 12]); +let ct = cipher.encrypt(&nonce, Payload { msg: plaintext, aad }).unwrap(); ``` ```rust @@ -76,10 +76,10 @@ cipher.encrypt(&nonce, aad, plaintext, &mut ct)?; use chacha20poly1305::{XChaCha20Poly1305, Key, XNonce, KeyInit}; use chacha20poly1305::aead::{Aead, Payload}; -let key = Key::from_slice(&[0u8; 32]); -let cipher = XChaCha20Poly1305::new(key); -let nonce = XNonce::from_slice(&[0u8; 24]); -let ct = cipher.encrypt(nonce, Payload { msg: plaintext, aad }).unwrap(); +let key = Key::from([0u8; 32]); +let cipher = XChaCha20Poly1305::new(&key); +let nonce = XNonce::from([0u8; 24]); +let ct = cipher.encrypt(&nonce, Payload { msg: plaintext, aad }).unwrap(); ``` ```rust diff --git a/docs/migration/RustCrypto/ed25519-dalek.md b/docs/migration/RustCrypto/ed25519-dalek.md index 14e28ac7..11e904e4 100644 --- a/docs/migration/RustCrypto/ed25519-dalek.md +++ b/docs/migration/RustCrypto/ed25519-dalek.md @@ -4,14 +4,14 @@ > `Ed25519SecretKey` / `Ed25519PublicKey` / `Ed25519Signature`. Signing > preserves RFC 8032 output bytes, and the rscrypto verifier is always strict. -Verified against `ed25519-dalek = "2.2.0"` and the `rscrypto` 0.7.8 line. +Verified against `ed25519-dalek = "3.0.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/ed25519_rfc8032_vectors.rs`, `tests/ed25519_oracle.rs`, and `tests/ed25519_wycheproof.rs`. ## TL;DR -| | Before (`ed25519-dalek` 2.x) | After (`rscrypto` 0.7.8) | +| | Before (`ed25519-dalek` 3.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `ed25519-dalek = "2.2"` | `rscrypto = { version = "0.7.8", features = ["ed25519"] }` | +| Cargo dep | `ed25519-dalek = "3.0"` | `rscrypto = { version = "0.8.1", features = ["ed25519"] }` | | Import | `use ed25519_dalek::{SigningKey, Signer, Verifier};` | `use rscrypto::{Ed25519SecretKey, Ed25519PublicKey, Ed25519Signature};` | | Sign | `signing_key.sign(msg)` | `secret.sign(msg)` | | Verify | `verifying_key.verify_strict(msg, &sig)?` | `public_key.verify(msg, &sig)?` | @@ -21,13 +21,13 @@ Evidence: `tests/ed25519_rfc8032_vectors.rs`, `tests/ed25519_oracle.rs`, and `te ```toml # Before [dependencies] -ed25519-dalek = "2.2" +ed25519-dalek = "3.0" ``` ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["ed25519"] } +rscrypto = { version = "0.8.1", features = ["ed25519"] } ``` The `ed25519` feature implies `sha2` (Ed25519 uses SHA-512 internally per RFC 8032). @@ -119,7 +119,7 @@ rscrypto's `verify` is *always* strict (rejects small-order public keys, non-can ### Keypair (combined secret + public) ```rust -// Before: no first-class Keypair type in dalek 2.x; use SigningKey directly. +// Before: no first-class Keypair type in dalek 3.x; use SigningKey directly. let signing_key = SigningKey::from_bytes(&seed); let pk = signing_key.verifying_key(); ``` diff --git a/docs/migration/RustCrypto/hkdf.md b/docs/migration/RustCrypto/hkdf.md index 7fe9290f..a54b6b52 100644 --- a/docs/migration/RustCrypto/hkdf.md +++ b/docs/migration/RustCrypto/hkdf.md @@ -2,14 +2,14 @@ > Replace `Hkdf::::new(Some(salt), ikm)` with `HkdfSha256::new(salt, ikm)`. The `Option<&[u8]>` salt becomes a plain `&[u8]` (empty slice == "no salt"); fused one-shot `HkdfSha256::derive_array::(...)` collapses extract+expand. -Verified against `hkdf = "0.13.0"` and the `rscrypto` 0.7.8 line. +Verified against `hkdf = "0.13.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/hkdf_sha256_vectors.rs`, `tests/hkdf_sha384_vectors.rs`, `tests/hkdf_sha512_vectors.rs`, the HKDF proptests, and `tests/hkdf_wycheproof.rs`. ## TL;DR -| | Before (`hkdf` 0.13.x) | After (`rscrypto` 0.7.8) | +| | Before (`hkdf` 0.13.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `hkdf = "0.13"` + `sha2 = "0.11"` | `rscrypto = { version = "0.7.8", features = ["hkdf"] }` | +| Cargo dep | `hkdf = "0.13"` + `sha2 = "0.11"` | `rscrypto = { version = "0.8.1", features = ["hkdf"] }` | | Import | `use hkdf::Hkdf; use sha2::Sha256;` | `use rscrypto::HkdfSha256;` | | Call | `Hkdf::::new(Some(salt), ikm).expand(info, &mut okm)?` | `HkdfSha256::new(salt, ikm).expand(info, &mut okm)?` | @@ -25,7 +25,7 @@ sha2 = "0.11" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["hkdf"] } +rscrypto = { version = "0.8.1", features = ["hkdf"] } ``` The `hkdf` feature implies `hmac` which implies `sha2`. diff --git a/docs/migration/RustCrypto/hmac.md b/docs/migration/RustCrypto/hmac.md index b037de9b..edc7745c 100644 --- a/docs/migration/RustCrypto/hmac.md +++ b/docs/migration/RustCrypto/hmac.md @@ -2,14 +2,14 @@ > Replace `Hmac::` / `Hmac::` (generic over digest) with named rscrypto types such as `HmacSha256` and `HmacSha3_256`. Key construction is infallible, `finalize()` borrows, and one-shot helpers return typed tags with sealed comparison decisions. -Verified against `hmac = "0.13.0"` and the `rscrypto` 0.7.8 line. +Verified against `hmac = "0.13.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/hmac_sha256_vectors.rs`, `tests/hmac_sha2_family_vectors.rs`, `tests/hmac_sha3_vectors.rs`, the HMAC proptests, and `tests/hmac_wycheproof.rs`. ## TL;DR -| | Before (`hmac` 0.13.x) | After (`rscrypto` 0.7.8) | +| | Before (`hmac` 0.13.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `hmac = "0.13"` + `sha2 = "0.11"` | `rscrypto = { version = "0.7.8", features = ["hmac"] }` | +| Cargo dep | `hmac = "0.13"` + `sha2 = "0.11"` | `rscrypto = { version = "0.8.1", features = ["hmac"] }` | | Import | `use hmac::{Hmac, Mac, KeyInit}; use sha2::Sha256;` | `use rscrypto::{HmacSha256, Mac};` | | Call | `Hmac::::new_from_slice(key).unwrap().chain_update(data).finalize().into_bytes()` | `HmacSha256::mac(key, data)` | @@ -25,7 +25,7 @@ sha2 = "0.11" # required as the generic parameter ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["hmac"] } +rscrypto = { version = "0.8.1", features = ["hmac"] } ``` The `hmac` feature implies `sha2`: no second dep to manage for SHA-2 HMAC. Use `features = ["hmac-sha3"]` for HMAC-SHA3, or `features = ["macs"]` when you want both SHA-2 and SHA-3 HMAC families. diff --git a/docs/migration/RustCrypto/p256.md b/docs/migration/RustCrypto/p256.md index affa3d7b..74ffe26c 100644 --- a/docs/migration/RustCrypto/p256.md +++ b/docs/migration/RustCrypto/p256.md @@ -13,9 +13,9 @@ Evidence: `tests/ecdsa_oracle.rs`, `src/auth/ecdsa.rs` unit tests, and ## TL;DR -| | Before (`p256` 0.14) | After (`rscrypto` 0.7.8) | +| | Before (`p256` 0.14) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `p256 = { version = "0.14", features = ["ecdsa"] }` | `rscrypto = { version = "0.7.8", default-features = false, features = ["ecdsa-p256"] }` | +| Cargo dep | `p256 = { version = "0.14", features = ["ecdsa"] }` | `rscrypto = { version = "0.8.1", default-features = false, features = ["ecdsa-p256"] }` | | Import | `use p256::ecdsa::{SigningKey, Signature};` | `use rscrypto::{EcdsaP256SecretKey, EcdsaP256PublicKey, EcdsaP256Signature};` | | Sign | `signing_key.sign(msg)` | `secret.try_sign(msg)?` | | Verify | `verifying_key.verify(msg, &sig)?` | `public.verify(msg, &sig)?` | @@ -27,7 +27,7 @@ Evidence: `tests/ecdsa_oracle.rs`, `src/auth/ecdsa.rs` unit tests, and p256 = { version = "0.14", features = ["ecdsa"] } # After -rscrypto = { version = "0.7.8", default-features = false, features = ["ecdsa-p256"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["ecdsa-p256"] } ``` The `ecdsa-p256` feature implies `hmac`, which is used for deterministic nonce diff --git a/docs/migration/RustCrypto/p384.md b/docs/migration/RustCrypto/p384.md index 3032e5c0..4f1328b8 100644 --- a/docs/migration/RustCrypto/p384.md +++ b/docs/migration/RustCrypto/p384.md @@ -6,16 +6,16 @@ rscrypto supports the fixed P-384/SHA-384 ECDSA profile, raw `r || s` signatures, DER signature import, SEC1/SPKI public-key import, deterministic signing, and caller-blinded signing. -Verified against RustCrypto `p384 0.13.1`. +Verified against RustCrypto `p384 0.14.0`. Evidence: `tests/ecdsa_oracle.rs`, `src/auth/ecdsa.rs` unit tests, and `fuzz/target_impls/auth_ecdsa_verify.rs` / `fuzz/target_impls/auth_ecdsa_sign.rs`. ## TL;DR -| | Before (`p384` 0.13) | After (`rscrypto` 0.7.8) | +| | Before (`p384` 0.14) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `p384 = { version = "0.13", features = ["ecdsa"] }` | `rscrypto = { version = "0.7.8", default-features = false, features = ["ecdsa-p384"] }` | +| Cargo dep | `p384 = { version = "0.14", features = ["ecdsa"] }` | `rscrypto = { version = "0.8.1", default-features = false, features = ["ecdsa-p384"] }` | | Import | `use p384::ecdsa::{SigningKey, Signature};` | `use rscrypto::{EcdsaP384SecretKey, EcdsaP384PublicKey, EcdsaP384Signature};` | | Sign | `signing_key.sign(msg)` | `secret.try_sign(msg)?` | | Verify | `verifying_key.verify(msg, &sig)?` | `public.verify(msg, &sig)?` | @@ -24,10 +24,10 @@ Evidence: `tests/ecdsa_oracle.rs`, `src/auth/ecdsa.rs` unit tests, and ```toml # Before -p384 = { version = "0.13", features = ["ecdsa"] } +p384 = { version = "0.14", features = ["ecdsa"] } # After -rscrypto = { version = "0.7.8", default-features = false, features = ["ecdsa-p384"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["ecdsa-p384"] } ``` The `ecdsa-p384` feature implies `hmac`, which is used for deterministic nonce diff --git a/docs/migration/RustCrypto/pbkdf2.md b/docs/migration/RustCrypto/pbkdf2.md index e7433ecb..4ac1a8ed 100644 --- a/docs/migration/RustCrypto/pbkdf2.md +++ b/docs/migration/RustCrypto/pbkdf2.md @@ -2,14 +2,14 @@ > Replace the free function `pbkdf2_hmac::(password, salt, iters, &mut out)` with `Pbkdf2Sha256::derive_key_array::(password, salt, iters)?`. The password helpers enforce the current PBKDF2 iteration and salt floors by default, while `*_primitive` APIs remain available for RFC vectors and legacy compatibility. -Verified against `pbkdf2 = "0.13.0"` and the `rscrypto` 0.7.8 line. +Verified against `pbkdf2 = "0.13.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/pbkdf2_kat_vectors.rs`, `tests/pbkdf2_differential.rs`, and `tests/pbkdf2_wycheproof.rs`. ## TL;DR -| | Before (`pbkdf2` 0.13.x) | After (`rscrypto` 0.7.8) | +| | Before (`pbkdf2` 0.13.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `pbkdf2 = "0.13"` + `sha2 = "0.11"` | `rscrypto = { version = "0.7.8", features = ["pbkdf2"] }` | +| Cargo dep | `pbkdf2 = "0.13"` + `sha2 = "0.11"` | `rscrypto = { version = "0.8.1", features = ["pbkdf2"] }` | | Import | `use pbkdf2::pbkdf2_hmac; use sha2::Sha256;` | `use rscrypto::Pbkdf2Sha256;` | | Call | `pbkdf2_hmac::(pw, salt, iters, &mut okm)` | `Pbkdf2Sha256::derive_key(pw, salt, iters, &mut okm)?` | @@ -25,7 +25,7 @@ sha2 = "0.11" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["pbkdf2"] } +rscrypto = { version = "0.8.1", features = ["pbkdf2"] } ``` The `pbkdf2` feature implies `hmac` which implies `sha2`. diff --git a/docs/migration/RustCrypto/rsa.md b/docs/migration/RustCrypto/rsa.md index f565ef57..7bcf1867 100644 --- a/docs/migration/RustCrypto/rsa.md +++ b/docs/migration/RustCrypto/rsa.md @@ -17,10 +17,10 @@ interoperability tests used by the stack migration guides. rsa = { version = "0.9", features = ["sha2"] } # After -rscrypto = { version = "0.7.8", default-features = false, features = ["rsa"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["rsa"] } # After, when generating keys or using randomized private/encryption APIs -rscrypto = { version = "0.7.8", default-features = false, features = ["rsa", "getrandom"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["rsa", "getrandom"] } ``` ## Map diff --git a/docs/migration/RustCrypto/scrypt.md b/docs/migration/RustCrypto/scrypt.md index ff188461..46668b11 100644 --- a/docs/migration/RustCrypto/scrypt.md +++ b/docs/migration/RustCrypto/scrypt.md @@ -11,10 +11,10 @@ The implementation is checked against RFC 7914 vectors and the RustCrypto `scryp ```toml # Raw scrypt KDF -rscrypto = { version = "0.7.8", default-features = false, features = ["scrypt"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["scrypt"] } # Password-record generation and verification with caller-owned entropy -rscrypto = { version = "0.7.8", default-features = false, features = [ +rscrypto = { version = "0.8.1", default-features = false, features = [ "scrypt", "phc-strings", ] } diff --git a/docs/migration/RustCrypto/sha2.md b/docs/migration/RustCrypto/sha2.md index 60f62560..116d77ff 100644 --- a/docs/migration/RustCrypto/sha2.md +++ b/docs/migration/RustCrypto/sha2.md @@ -2,14 +2,14 @@ > Same algorithms (FIPS 180-4), same `Sha224` / `Sha256` / `Sha384` / `Sha512` / `Sha512_256` names, same `new()` / `update()` / `finalize()` shape. The output type changes from `Output` (a `GenericArray`) to `[u8; N]`, and `finalize()` now borrows instead of consuming. -Verified against `sha2 = "0.11.0"` and the `rscrypto` 0.7.8 line. +Verified against `sha2 = "0.11.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/sha2_official_vectors.rs`, `tests/sha256_differential.rs`, and `tests/sha512_differential.rs`. ## TL;DR -| | Before (`sha2` 0.11.x) | After (`rscrypto` 0.7.8) | +| | Before (`sha2` 0.11.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `sha2 = "0.11"` | `rscrypto = { version = "0.7.8", features = ["sha2"] }` | +| Cargo dep | `sha2 = "0.11"` | `rscrypto = { version = "0.8.1", features = ["sha2"] }` | | Import | `use sha2::{Sha256, Digest};` | `use rscrypto::Sha256;` | | Call | `Sha256::digest(data)` | `Sha256::digest(data)` | @@ -24,7 +24,7 @@ sha2 = "0.11" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["sha2"] } +rscrypto = { version = "0.8.1", features = ["sha2"] } ``` ## Algorithm map diff --git a/docs/migration/RustCrypto/sha3.md b/docs/migration/RustCrypto/sha3.md index b2b59e0a..e27ce7f1 100644 --- a/docs/migration/RustCrypto/sha3.md +++ b/docs/migration/RustCrypto/sha3.md @@ -2,14 +2,14 @@ > Same algorithms (FIPS 202: SHA3-224/256/384/512, SHAKE128/256; SP 800-185: cSHAKE128/256), `[u8; N]` outputs replace `Output`, and the SHAKE/cSHAKE reader chain uses `squeeze(&mut out)`. -Verified against `sha3 = "0.12.0"` and the `rscrypto` 0.7.8 line. +Verified against `sha3 = "0.12.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/sha3_official_vectors.rs`, `tests/sha3_differential.rs`, `tests/shake128_differential.rs`, `tests/shake256_differential.rs`, `tests/cshake256_nist_vectors.rs`, and `tests/cshake256_differential.rs`. ## TL;DR -| | Before (`sha3` 0.12.x) | After (`rscrypto` 0.7.8) | +| | Before (`sha3` 0.12.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `sha3 = "0.12"` | `rscrypto = { version = "0.7.8", features = ["sha3"] }` | +| Cargo dep | `sha3 = "0.12"` | `rscrypto = { version = "0.8.1", features = ["sha3"] }` | | Import | `use sha3::{Sha3_256, Digest};` | `use rscrypto::{Sha3_256, Digest};` | | Call | `Sha3_256::digest(data)` | `Sha3_256::digest(data)` | @@ -24,7 +24,7 @@ sha3 = "0.12" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["sha3"] } +rscrypto = { version = "0.8.1", features = ["sha3"] } ``` ## Algorithm map diff --git a/docs/migration/RustCrypto/x25519-dalek.md b/docs/migration/RustCrypto/x25519-dalek.md index b2e21aff..5f8590b0 100644 --- a/docs/migration/RustCrypto/x25519-dalek.md +++ b/docs/migration/RustCrypto/x25519-dalek.md @@ -5,14 +5,14 @@ > `X25519SharedSecret`. Valid RFC 7748 inputs produce the same shared-secret > bytes; rscrypto returns an explicit error for an all-zero result. -Verified against `x25519-dalek = "2.0.1"` and the `rscrypto` 0.7.8 line. +Verified against `x25519-dalek = "3.0.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/x25519_vectors.rs`, `tests/x25519_oracle.rs`, and `tests/x25519_wycheproof.rs`. ## TL;DR -| | Before (`x25519-dalek` 2.x) | After (`rscrypto` 0.7.8) | +| | Before (`x25519-dalek` 3.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `x25519-dalek = { version = "2.0", features = ["static_secrets"] }` | `rscrypto = { version = "0.7.8", features = ["x25519"] }` | +| Cargo dep | `x25519-dalek = { version = "3.0", features = ["static_secrets"] }` | `rscrypto = { version = "0.8.1", features = ["x25519"] }` | | Import | `use x25519_dalek::{StaticSecret, PublicKey};` | `use rscrypto::{X25519SecretKey, X25519PublicKey};` | | DH | `secret.diffie_hellman(&peer_pub)` (returns `SharedSecret`) | `secret.diffie_hellman(&peer_pub)?` (returns `Result`) | @@ -21,13 +21,13 @@ Evidence: `tests/x25519_vectors.rs`, `tests/x25519_oracle.rs`, and `tests/x25519 ```toml # Before [dependencies] -x25519-dalek = { version = "2.0", features = ["static_secrets"] } +x25519-dalek = { version = "3.0", features = ["static_secrets"] } ``` ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["x25519"] } +rscrypto = { version = "0.8.1", features = ["x25519"] } ``` The `x25519` feature has no transitive dependencies: X25519 needs nothing beyond Curve25519 arithmetic. diff --git a/docs/migration/aegis.md b/docs/migration/aegis.md index 83796170..fd8f771c 100644 --- a/docs/migration/aegis.md +++ b/docs/migration/aegis.md @@ -4,14 +4,14 @@ > rscrypto's caller-buffer `encrypt(&nonce, aad, msg, &mut out)`. The mapped > AEGIS-256 operation preserves ciphertext and tag bytes. -Verified against `aegis = "0.9.12"` and the `rscrypto` 0.7.8 line. +Verified against `aegis = "0.9.15"` and the `rscrypto` 0.8.1 line. Evidence: `tests/aegis256_oracle.rs` and `tests/aead_wycheproof.rs`. ## TL;DR -| | Before (`aegis` 0.9.x) | After (`rscrypto` 0.7.8) | +| | Before (`aegis` 0.9.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `aegis = "0.9"` | `rscrypto = { version = "0.7.8", features = ["aegis256"] }` | +| Cargo dep | `aegis = "0.9"` | `rscrypto = { version = "0.8.1", features = ["aegis256"] }` | | Import | `use aegis::aegis256::Aegis256;` | `use rscrypto::{Aead, Aegis256, Aegis256Key, aead::{Nonce256, expert::AeadWithNonce}};` | | Encrypt | `Aegis256::<16>::new(&key, &nonce).encrypt(msg, aad) -> (Vec, [u8; 16])` | `cipher.encrypt(&nonce, aad, msg, &mut out)?` | @@ -26,7 +26,7 @@ aegis = "0.9" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["aegis256"] } +rscrypto = { version = "0.8.1", features = ["aegis256"] } ``` ## Algorithm map diff --git a/docs/migration/api-boundaries.md b/docs/migration/api-boundaries.md index 5ceb929c..74e02204 100644 --- a/docs/migration/api-boundaries.md +++ b/docs/migration/api-boundaries.md @@ -40,6 +40,28 @@ protocol defines a shorter value. Panicking `random()` constructors were removed. Use `try_random()` or the type-specific `try_generate()` method and propagate entropy failures. +## RSA caller-supplied blinding + +Caller-supplied RSA blinding methods now take one borrowed `RsaBlindingPair` +instead of adjacent factor and inverse slices: + +```rust +use rscrypto::{RsaBlindingPair, RsaPkcs1v15Profile}; + +let blinding = RsaBlindingPair::new(blinding_factor, blinding_factor_inverse); +private_key.sign_pkcs1v15_with_blinding_factor( + RsaPkcs1v15Profile::Sha256, + message, + blinding, + signature, +)?; +# Ok::<(), rscrypto::RsaPrivateOpError>(()) +``` + +The pair borrows both inputs without allocation or copying their bytes. Every +accepting operation still validates their width, range, and inverse relation +against its RSA key. Error variants and output clearing are unchanged. + ## AEAD nonces Normal `Aead` sealing generates a fresh OS nonce. AES-GCM also supports the diff --git a/docs/migration/aws-lc-rs.md b/docs/migration/aws-lc-rs.md index a8fab39c..261b7f99 100644 --- a/docs/migration/aws-lc-rs.md +++ b/docs/migration/aws-lc-rs.md @@ -12,7 +12,7 @@ The direct examples below are covered by `tests/migration_aws_lc_rs.rs`. aws-lc-rs = "1.17" # After: enable only the primitives you use -rscrypto = { version = "0.7.8", default-features = false, features = ["sha2", "hmac", "hkdf", "pbkdf2", "aes-gcm", "chacha20poly1305", "ecdsa", "ed25519", "x25519", "rsa"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["sha2", "hmac", "hkdf", "pbkdf2", "aes-gcm", "chacha20poly1305", "ecdsa", "ed25519", "x25519", "rsa"] } ``` ## Map diff --git a/docs/migration/blake3.md b/docs/migration/blake3.md index 86b7e509..8e70bd1b 100644 --- a/docs/migration/blake3.md +++ b/docs/migration/blake3.md @@ -4,14 +4,14 @@ > Unkeyed hashes return `[u8; 32]`; keyed hashes return `Blake3KeyedHash` so > authenticator comparison requires an explicit sealed decision. -Verified against `blake3 = "1.8.5"` and the `rscrypto` 0.7.8 line. +Verified against `blake3 = "1.8.6"` and the `rscrypto` 0.8.1 line. Evidence: `tests/blake3_official_vectors.rs` and `tests/blake3_differential.rs`. ## TL;DR -| | Before (`blake3` 1.x) | After (`rscrypto` 0.7.8) | +| | Before (`blake3` 1.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `blake3 = "1.8"` | `rscrypto = { version = "0.7.8", features = ["blake3"] }` | +| Cargo dep | `blake3 = "1.8"` | `rscrypto = { version = "0.8.1", features = ["blake3"] }` | | Import | `use blake3::Hasher;` | `use rscrypto::{Blake3, prelude::*};` | | Call | `blake3::hash(data).as_bytes()` | `&Blake3::digest(data)` | @@ -26,7 +26,7 @@ blake3 = "1.8" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["blake3"] } +rscrypto = { version = "0.8.1", features = ["blake3"] } ``` ## API patterns diff --git a/docs/migration/crc-fast.md b/docs/migration/crc-fast.md index dc0bf553..d2b791f8 100644 --- a/docs/migration/crc-fast.md +++ b/docs/migration/crc-fast.md @@ -4,15 +4,15 @@ Replace `crc-fast`'s enum-driven `checksum(CrcAlgorithm::*, data)` calls with rscrypto's named CRC types. The mapped algorithms keep the same CRC parameters, but rscrypto returns each algorithm's natural integer width instead of `u64`. -Verified against `crc-fast = "1.10.0"` and the `rscrypto` 0.7.8 line. +Verified against `crc-fast = "1.10.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/crc16_properties.rs`, `tests/crc32_properties.rs`, and `tests/crc64_properties.rs`. ## TL;DR -| | Before (`crc-fast` 1.x) | After (`rscrypto` 0.7.8) | +| | Before (`crc-fast` 1.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `crc-fast = "1.10"` | `rscrypto = { version = "0.7.8", features = ["crc32", "crc64"] }` | +| Cargo dep | `crc-fast = "1.10"` | `rscrypto = { version = "0.8.1", features = ["crc32", "crc64"] }` | | Import | `use crc_fast::{checksum, CrcAlgorithm};` | `use rscrypto::checksum::{Checksum, Crc32};` | | Call | `checksum(CrcAlgorithm::Crc32IsoHdlc, data) as u32` | `Crc32::checksum(data)` | @@ -27,7 +27,7 @@ crc-fast = "1.10" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["crc32", "crc64"] } +rscrypto = { version = "0.8.1", features = ["crc32", "crc64"] } ``` Add `crc16` or `crc24` only when you migrate a mapped variant from those diff --git a/docs/migration/crc.md b/docs/migration/crc.md index 91b06595..89102218 100644 --- a/docs/migration/crc.md +++ b/docs/migration/crc.md @@ -4,14 +4,14 @@ Replace supported `Crc::::new(&CRC_*)` configurations with rscrypto's named CRC types. The mapped algorithms retain their output parameters and have a portable fallback plus selected target-specific backends. -Verified against `crc = "3.4.0"` and the `rscrypto` 0.7.8 line. +Verified against `crc = "3.4.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/crc16_properties.rs`, `tests/crc24_properties.rs`, `tests/crc32_properties.rs`, and `tests/crc64_properties.rs`. ## TL;DR -| | Before (`crc` 3.x) | After (`rscrypto` 0.7.8) | +| | Before (`crc` 3.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `crc = "3.4"` | `rscrypto = { version = "0.7.8", features = ["checksums"] }` | +| Cargo dep | `crc = "3.4"` | `rscrypto = { version = "0.8.1", features = ["checksums"] }` | | Import | `use crc::{Crc, CRC_32_ISO_HDLC};` | `use rscrypto::checksum::{Checksum, Crc32};` | | Call | `Crc::::new(&CRC_32_ISO_HDLC).checksum(data)` | `Crc32::checksum(data)` | @@ -26,7 +26,7 @@ crc = "3.4" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["checksums"] } +rscrypto = { version = "0.8.1", features = ["checksums"] } ``` `features = ["checksums"]` enables every CRC family. To trim the build, pick only what you use: `crc16`, `crc24`, `crc32` (covers IEEE + Castagnoli), `crc64` (covers XZ + NVME). diff --git a/docs/migration/crc32c.md b/docs/migration/crc32c.md index 6c65db57..8f5d3fc3 100644 --- a/docs/migration/crc32c.md +++ b/docs/migration/crc32c.md @@ -9,9 +9,9 @@ Output is covered by the CRC-32C oracle/property tests in `tests/crc32_propertie ## TL;DR -| | Before (`crc32c` 0.6.x) | After (`rscrypto` 0.7.8) | +| | Before (`crc32c` 0.6.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `crc32c = "0.6"` | `rscrypto = { version = "0.7.8", features = ["crc32"] }` | +| Cargo dep | `crc32c = "0.6"` | `rscrypto = { version = "0.8.1", features = ["crc32"] }` | | Import | `use crc32c::{crc32c, crc32c_append, crc32c_combine};` | `use rscrypto::checksum::{Checksum, ChecksumCombine, Crc32C};` | | Call | `crc32c(data)` | `Crc32C::checksum(data)` | @@ -26,7 +26,7 @@ crc32c = "0.6" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["crc32"] } +rscrypto = { version = "0.8.1", features = ["crc32"] } ``` The `crc32` feature exposes both `Crc32` (IEEE) and `Crc32C` (Castagnoli). diff --git a/docs/migration/crc32fast.md b/docs/migration/crc32fast.md index 1aa5b225..e5768a69 100644 --- a/docs/migration/crc32fast.md +++ b/docs/migration/crc32fast.md @@ -6,9 +6,9 @@ Output is covered by the CRC-32 oracle/property tests in `tests/crc32_properties ## TL;DR -| | Before (`crc32fast` 1.x) | After (`rscrypto` 0.7.8) | +| | Before (`crc32fast` 1.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `crc32fast = "1.5"` | `rscrypto = { version = "0.7.8", features = ["crc32"] }` | +| Cargo dep | `crc32fast = "1.5"` | `rscrypto = { version = "0.8.1", features = ["crc32"] }` | | Import | `use crc32fast::Hasher;` | `use rscrypto::checksum::{Checksum, Crc32};` | | Call | `crc32fast::hash(data)` | `Crc32::checksum(data)` | @@ -23,7 +23,7 @@ crc32fast = "1.5" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["crc32"] } +rscrypto = { version = "0.8.1", features = ["crc32"] } ``` `features = ["crc32"]` exposes both CRC-32/IEEE (`Crc32`) and diff --git a/docs/migration/crc64fast.md b/docs/migration/crc64fast.md index 67dedab3..1b3e3622 100644 --- a/docs/migration/crc64fast.md +++ b/docs/migration/crc64fast.md @@ -2,14 +2,14 @@ > Same CRC-64/XZ output with method renames from `Digest` / `write` / `sum64` to `Crc64` / `update` / `finalize`. The aside below also covers `crc64fast-nvme` → `Crc64Nvme`. -Verified against `crc64fast = "1.1.0"` for `Crc64`; `Crc64Nvme` oracle coverage uses `crc-fast = "1.10.0"` and the `rscrypto` 0.7.8 line. +Verified against `crc64fast = "1.1.0"` for `Crc64`; `Crc64Nvme` oracle coverage uses `crc-fast = "1.10.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/crc64_properties.rs` compares one-shot, streaming, and combine output against those oracle crates. ## TL;DR -| | Before (`crc64fast` 1.x) | After (`rscrypto` 0.7.8) | +| | Before (`crc64fast` 1.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `crc64fast = "1.1"` | `rscrypto = { version = "0.7.8", features = ["crc64"] }` | +| Cargo dep | `crc64fast = "1.1"` | `rscrypto = { version = "0.8.1", features = ["crc64"] }` | | Import | `use crc64fast::Digest;` | `use rscrypto::checksum::{Checksum, Crc64};` | | Call | `Digest::new(); .write(data); .sum64()` | `Crc64::new(); .update(data); .finalize()` | @@ -24,7 +24,7 @@ crc64fast = "1.1" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["crc64"] } +rscrypto = { version = "0.8.1", features = ["crc64"] } ``` `features = ["crc64"]` enables both `Crc64` (XZ / ECMA-182) and `Crc64Nvme`. @@ -120,7 +120,7 @@ Drop both `crc64fast` and `crc64fast-nvme` from Cargo.toml; `features = ["crc64" also matter. Do not substitute CRC-64/ISO. - **No reset, no resume in `crc64fast`.** Build a fresh `Digest` per checksum. rscrypto adds `.reset()` and `Crc64::resume(prev)` on top of the same shape. - **`no_std`.** `crc64fast` requires `std` for SIMD detection. rscrypto's `Crc64` is `no_std`-capable; runtime detection is gated on the `std` feature, with compile-time `target_feature` selection in `no_std` builds and a portable fallback always present. -- **Hardware coverage.** `crc64fast` ships x86_64 (PCLMUL) and aarch64 (PMULL) backends. rscrypto adds VPCLMULQDQ (large buffers on x86_64), SVE2 PMULL (aarch64), VPMSUMD (Power), VGFM (s390x), and Zbc/Zvbc (RISC-V). +- **Hardware coverage.** `crc64fast` ships x86_64 (PCLMUL) and aarch64 (PMULL) backends. rscrypto adds VPCLMULQDQ (large buffers on x86_64), SVE2 PMULL (aarch64), VPMSUMD (Power), and VGFM (s390x). RISC-V uses the portable slice-by-16 implementation. - **Force a backend.** `RSCRYPTO_CRC64_FORCE=portable` selects the portable CRC-64 runtime backend in `std` builds. The `portable-only` feature makes runtime capability detection ignore host acceleration; see diff --git a/docs/migration/dryoc.md b/docs/migration/dryoc.md index a134aad4..031710dd 100644 --- a/docs/migration/dryoc.md +++ b/docs/migration/dryoc.md @@ -14,7 +14,7 @@ The direct BLAKE2b, Ed25519, and X25519 examples below are covered by dryoc = "0.8" # After: choose only the primitive features you use -rscrypto = { version = "0.7.8", default-features = false, features = ["blake2b", "ed25519", "x25519", "argon2"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["blake2b", "ed25519", "x25519", "argon2"] } ``` ## Map diff --git a/docs/migration/openssl.md b/docs/migration/openssl.md index efbe7671..3ca47e4e 100644 --- a/docs/migration/openssl.md +++ b/docs/migration/openssl.md @@ -15,7 +15,7 @@ RSA test lane. openssl = "0.10" # After: choose only the primitive features you use -rscrypto = { version = "0.7.8", default-features = false, features = ["sha2", "hmac", "hkdf", "pbkdf2", "aes-gcm", "chacha20poly1305", "ed25519", "x25519", "rsa"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["sha2", "hmac", "hkdf", "pbkdf2", "aes-gcm", "chacha20poly1305", "ed25519", "x25519", "rsa"] } ``` ## Map diff --git a/docs/migration/rapidhash.md b/docs/migration/rapidhash.md index 4c346ea2..2172298e 100644 --- a/docs/migration/rapidhash.md +++ b/docs/migration/rapidhash.md @@ -8,7 +8,7 @@ inherent methods can run at compile time. ```toml [dependencies] -rscrypto = { version = "0.7.8", features = ["rapidhash"] } +rscrypto = { version = "0.8.1", features = ["rapidhash"] } ``` ```rust diff --git a/docs/migration/ring.md b/docs/migration/ring.md index d6664eec..2600a697 100644 --- a/docs/migration/ring.md +++ b/docs/migration/ring.md @@ -13,7 +13,7 @@ The direct examples below are covered by `tests/migration_ring.rs`. ring = "0.17" # After: choose only the primitive features you use -rscrypto = { version = "0.7.8", default-features = false, features = ["sha2", "hmac", "hkdf", "pbkdf2", "aes-gcm", "chacha20poly1305", "ecdsa", "ed25519", "rsa"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["sha2", "hmac", "hkdf", "pbkdf2", "aes-gcm", "chacha20poly1305", "ecdsa", "ed25519", "rsa"] } ``` ## Map diff --git a/docs/migration/sha3-kmac.md b/docs/migration/sha3-kmac.md index 47ddf764..97a13aa2 100644 --- a/docs/migration/sha3-kmac.md +++ b/docs/migration/sha3-kmac.md @@ -13,9 +13,9 @@ Evidence: `tests/kmac128_nist_vectors.rs`, `tests/kmac128_differential.rs`, ## TL;DR -| | Before (`sha3-kmac` 0.3.x) | After (`rscrypto` 0.7.8) | +| | Before (`sha3-kmac` 0.3.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `sha3-kmac = "0.3"` | `rscrypto = { version = "0.7.8", features = ["kmac"] }` | +| Cargo dep | `sha3-kmac = "0.3"` | `rscrypto = { version = "0.8.1", features = ["kmac"] }` | | Import | `use sha3_kmac::Kmac256;` | `use rscrypto::Kmac256;` | | Call | `let mut k = Kmac256::new(key, custom)?; k.update(data); k.finalize_into(&mut tag);` | `Kmac256::mac_into(key, custom, data, &mut tag);` | @@ -30,7 +30,7 @@ sha3-kmac = "0.3" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["kmac"] } +rscrypto = { version = "0.8.1", features = ["kmac"] } ``` The `kmac` feature implies `sha3`. diff --git a/docs/migration/tiny-keccak.md b/docs/migration/tiny-keccak.md index e22ab527..4f96756a 100644 --- a/docs/migration/tiny-keccak.md +++ b/docs/migration/tiny-keccak.md @@ -5,14 +5,14 @@ Replace fixed-output `tiny_keccak::Kmac::v128` / `Kmac::v256` and `Kmac256` and `rscrypto::Cshake128` / `Cshake256`. KMAC construction is infallible and adds verification helpers. `KmacXof` is not mapped. -Verified against `tiny-keccak = "2.0.2"` (with `kmac` and `cshake` features) and the `rscrypto` 0.7.8 line. +Verified against `tiny-keccak = "2.0.2"` (with `kmac` and `cshake` features) and the `rscrypto` 0.8.1 line. Evidence: `tests/kmac128_differential.rs`, `tests/kmac256_differential.rs`, `tests/cshake256_differential.rs`, `tests/cshake256_nist_vectors.rs`, and `tests/kmac_wycheproof.rs`. ## TL;DR -| | Before (`tiny-keccak` 2.x) | After (`rscrypto` 0.7.8) | +| | Before (`tiny-keccak` 2.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `tiny-keccak = { version = "2.0", features = ["kmac", "cshake"] }` | `rscrypto = { version = "0.7.8", features = ["kmac"] }` | +| Cargo dep | `tiny-keccak = { version = "2.0", features = ["kmac", "cshake"] }` | `rscrypto = { version = "0.8.1", features = ["kmac"] }` | | KMAC import | `use tiny_keccak::{Hasher, Kmac};` | `use rscrypto::{Kmac128, Kmac256};` | | KMAC call | `let mut k = Kmac::v256(key, custom); k.update(data); k.finalize(&mut tag);` | `Kmac256::mac_into(key, custom, data, &mut tag);` | | cSHAKE import | `use tiny_keccak::{Hasher, CShake};` | `use rscrypto::{Cshake128, Cshake256, Xof};` | @@ -29,7 +29,7 @@ tiny-keccak = { version = "2.0", features = ["kmac", "cshake"] } ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["kmac"] } +rscrypto = { version = "0.8.1", features = ["kmac"] } ``` The `kmac` feature implies `sha3` (which provides the underlying `Cshake128` / `Cshake256` sponges for both KMAC variants and the standalone cSHAKE primitives). diff --git a/docs/migration/twox-hash.md b/docs/migration/twox-hash.md index 686a0d55..32fb95f4 100644 --- a/docs/migration/twox-hash.md +++ b/docs/migration/twox-hash.md @@ -7,9 +7,9 @@ The `twox-hash` mapping is API migration guidance for its 2.x XXH3 surface. ## TL;DR -| | Before (`twox-hash` 2.x) | After (`rscrypto` 0.7.8) | +| | Before (`twox-hash` 2.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `twox-hash = "2.1"` | `rscrypto = { version = "0.7.8", features = ["xxh3"] }` | +| Cargo dep | `twox-hash = "2.1"` | `rscrypto = { version = "0.8.1", features = ["xxh3"] }` | | Import | `use twox_hash::XxHash3_64;` | `use rscrypto::{FastHash, Xxh3};` | | Call | `XxHash3_64::oneshot(data)` | `Xxh3::hash(data)` | @@ -24,7 +24,7 @@ twox-hash = "2.1" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["xxh3"] } +rscrypto = { version = "0.8.1", features = ["xxh3"] } ``` ## Algorithm map diff --git a/docs/migration/xxhash-rust.md b/docs/migration/xxhash-rust.md index 90097fe4..20e056ac 100644 --- a/docs/migration/xxhash-rust.md +++ b/docs/migration/xxhash-rust.md @@ -2,14 +2,14 @@ > Replace `xxhash_rust::xxh3::xxh3_64` with `rscrypto::Xxh3::hash` (and `xxh3_128` with `Xxh3_128::hash`). One-shot bit-equivalent. Streaming routes through `Xxh3Hasher` (a `core::hash::Hasher`). -Verified against `xxhash-rust = "0.8.16"` (with `xxh3` feature) and the `rscrypto` 0.7.8 line. +Verified against `xxhash-rust = "0.8.18"` (with `xxh3` feature) and the `rscrypto` 0.8.1 line. Evidence: `tests/xxh3_differential.rs`. ## TL;DR -| | Before (`xxhash-rust` 0.8.x) | After (`rscrypto` 0.7.8) | +| | Before (`xxhash-rust` 0.8.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `xxhash-rust = { version = "0.8", features = ["xxh3"] }` | `rscrypto = { version = "0.7.8", features = ["xxh3"] }` | +| Cargo dep | `xxhash-rust = { version = "0.8", features = ["xxh3"] }` | `rscrypto = { version = "0.8.1", features = ["xxh3"] }` | | Import | `use xxhash_rust::xxh3::xxh3_64;` | `use rscrypto::{FastHash, Xxh3};` | | Call | `xxh3_64(data)` | `Xxh3::hash(data)` | @@ -24,7 +24,7 @@ xxhash-rust = { version = "0.8", features = ["xxh3"] } ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["xxh3"] } +rscrypto = { version = "0.8.1", features = ["xxh3"] } ``` ## Algorithm map diff --git a/docs/secret-ownership.md b/docs/secret-ownership.md index 54eaf624..a9b69c34 100644 --- a/docs/secret-ownership.md +++ b/docs/secret-ownership.md @@ -43,7 +43,8 @@ capability for permanent retention. | ML-KEM prepared decapsulation keys | Explicit duplicate; no `Clone` or `Copy` | Masked | `SecretBytes` export; no Serde | Inline | Reuse of validated private arithmetic without making implicit copies | | `RsaPrivateKey` | Neither | Public key plus a redacted private-components field | PKCS#1/PKCS#8 DER into `SecretVec`; no Serde | Heap-backed big integers and Montgomery state | Standards-compatible private-key storage/export while keeping the returned allocation typed as secret | | `RsaPrivateScratch` | Neither | Public sizing metadata only | None | Reusable heap buffers and limb vectors | Amortizes private-operation allocation while keeping intermediate ownership with the caller | -| `RsaPrivateKeyParts<'a>` | `Clone + Copy` | No `Debug` | Borrowed import fields; no Serde | Borrowed | Pass-by-value import description; copying duplicates references, not private bytes | +| `RsaPrivateKeyParts<'a>` | `Clone + Copy` | Modulus size and public exponent only; private fields masked | Borrowed import fields; no Serde | Borrowed | Pass-by-value import description; copying duplicates references, not private bytes | +| `RsaBlindingPair<'a>` | `Clone + Copy` | Factor and inverse masked | Borrowed factor and inverse; no Serde | Borrowed | Couples the two caller-blinding inputs; private operations validate them against the selected key | | `RsaSignatureSigner<'a>` | `Clone + Copy` | Signature profile only | None | Borrowed | Reusable profile-bound signing handle; copying duplicates a private-key reference, not the key | | HMAC-SHA-2 and HMAC-SHA-3 states | Neither | Masked | None | Inline | Reuse through `reset`; independent streamed owners require explicit keyed construction rather than an implicit state copy | | HKDF-SHA-2 states | Neither | Masked | None | Inline | Repeated expansion borrows one extracted PRK owner, which can also be shared by reference | diff --git a/docs/test-vector-coverage.md b/docs/test-vector-coverage.md index f905818a..5882e3cf 100644 --- a/docs/test-vector-coverage.md +++ b/docs/test-vector-coverage.md @@ -65,7 +65,7 @@ the concrete inputs and outputs of each cryptography API. | XChaCha20-Poly1305 | `tests/xchacha20poly1305.rs` | `tests/aead_wycheproof.rs` covers Wycheproof 192-bit nonce open failure; unit/integration tests cover wrong nonce/tag/AAD | Current suite maps directly after nonce-size filtering | | AEGIS-256 | `tests/aegis256_oracle.rs` | `tests/aead_wycheproof.rs` covers Wycheproof AEGIS-256 open failure; unit/integration tests cover wrong nonce/tag/AAD | Current suite maps directly | | Ascon-AEAD128 | Final SP 800-232 reference corpus in `tests/ascon_final_kats.rs` and `testdata/ascon/asconaead128.txt`; independent coverage in `tests/ascon_aead_oracle.rs` | Unit/integration tests cover wrong nonce/tag/AAD and oracle decrypt failure | Current Wycheproof `ASCON128` vectors do not match this crate's NIST Ascon-AEAD128 variant, so they are not vendored | -| ECDSA P-256/P-384 signing and verification | `tests/ecdsa_oracle.rs`; `tests/ecdsa_wycheproof.rs`; `src/auth/ecdsa.rs` unit tests; RustCrypto `p256 0.14.0` / `p384 0.13.1` oracles; `fuzz/target_impls/auth_ecdsa_verify.rs`; `fuzz/target_impls/auth_ecdsa_sign.rs` | Current Wycheproof P-256/SHA-256 and P-384/SHA-384 DER suites cover valid and invalid verification cases; unit, oracle, and fuzz tests cover SEC1/SPKI/DER/raw parser boundaries, zero/out-of-range scalars and coordinates, tampered signatures, wrong messages, deterministic and blinded signing, low-S normalization, and public-key derivation | CT evidence covers blinded signing. Public verification remains public-input work unless promoted by the CT manifest. Variable-length P1363 parser cases do not map to the fixed-size raw signature APIs. | +| ECDSA P-256/P-384 signing and verification | `tests/ecdsa_oracle.rs`; `tests/ecdsa_wycheproof.rs`; `src/auth/ecdsa.rs` unit tests; RustCrypto `p256 0.14.0` / `p384 0.14.0` oracles; `fuzz/target_impls/auth_ecdsa_verify.rs`; `fuzz/target_impls/auth_ecdsa_sign.rs` | Current Wycheproof P-256/SHA-256 and P-384/SHA-384 DER suites cover valid and invalid verification cases; unit, oracle, and fuzz tests cover SEC1/SPKI/DER/raw parser boundaries, zero/out-of-range scalars and coordinates, tampered signatures, wrong messages, deterministic and blinded signing, low-S normalization, and public-key derivation | CT evidence covers blinded signing. Public verification remains public-input work unless promoted by the CT manifest. Variable-length P1363 parser cases do not map to the fixed-size raw signature APIs. | | Ed25519 | `tests/ed25519_rfc8032_vectors.rs`, `tests/ed25519_oracle.rs` | `tests/ed25519_wycheproof.rs` covers Wycheproof valid/invalid signatures and invalid public/signature encodings; unit tests cover small-order and non-canonical signatures | Current suite maps directly | | X25519 | `tests/x25519_vectors.rs`, `tests/x25519_oracle.rs` | `tests/x25519_wycheproof.rs` covers Wycheproof valid/acceptable XDH vectors and rejects all-zero shared secrets; RFC low-order and non-canonical public cases remain in `tests/x25519_vectors.rs` | ASN/JWK/PEM suites do not apply to byte-array API | | ML-KEM-512/768/1024 | `tests/mlkem_acvp.rs` covers NIST ACVP FIPS 203 keyGen, encapsulation, decapsulation, decapsulationKeyCheck, and encapsulationKeyCheck vectors for all parameter sets; `tests/mlkem_properties.rs` differentials arbitrary seeds against the `fips203` crate; `tests/mlkem_types.rs` checks FIPS 203 sizes, randomness, security categories, byte wrappers, secret redaction, public-value equality, and sealed secret-owner comparison | `tests/mlkem_ops.rs` covers non-canonical public-key rejection before randomness, prepared-key parity, prepared-key invalid material, wrong-length parsers, decapsulation-key hash mismatch, and modified-ciphertext implicit rejection; `fuzz/target_impls/auth_mlkem512.rs`, `auth_mlkem768.rs`, and `auth_mlkem1024.rs` cover round trips, parser inputs, and modified ciphertexts | No vendored Wycheproof ML-KEM suite is currently mapped; official ACVP vectors plus all-profile FIPS 203 differential/property coverage are the primary oracle set | diff --git a/docs/types.md b/docs/types.md index c5d913fa..5480ba5e 100644 --- a/docs/types.md +++ b/docs/types.md @@ -144,7 +144,7 @@ Features: `signatures` / `key-exchange` or `ecdsa` / `ed25519` / `rsa` / `x25519 | `EcdsaP256Keypair` / `EcdsaP384Keypair` | secret + public | FIPS 186-5 / SEC 1 | | `Ed25519SecretKey` / `Ed25519PublicKey` / `Ed25519Signature` | 32/32/64B | RFC 8032 | | `Ed25519Keypair` | -- | RFC 8032 | -| `RsaPublicKey`, `RsaPrivateKey`, `RsaPrivateKeyParts`, `RsaX509PublicKey`, `RsaPublicScratch`, `RsaPrivateScratch` | variable | RFC 8017 / RFC 4055 | +| `RsaPublicKey`, `RsaPrivateKey`, `RsaPrivateKeyParts`, `RsaBlindingPair`, `RsaX509PublicKey`, `RsaPublicScratch`, `RsaPrivateScratch` | variable | RFC 8017 / RFC 4055 | | `RsaSignatureSigner`, `RsaSignatureVerifier` | profile-bound wrappers | RFC 8017 / RFC 4055 | | `RsaJwtAlgorithm`, `RsaJwtVerifier` | verifier-owned JWT/JWS policy | RFC 7515 / RFC 8725 | | `RsaSignatureProfile`, `RsaPssProfile`, `RsaPkcs1v15Profile`, `RsaOaepProfile`, `RsaPublicKeyPolicy`, `RsaKeyGenerationContract` | -- | RFC 8017 / RFC 4055 / FIPS 186-5 / protocol-specific profiles | diff --git a/examples/aead_seal_open.rs b/examples/aead_seal_open.rs index 4484efee..249756d0 100644 --- a/examples/aead_seal_open.rs +++ b/examples/aead_seal_open.rs @@ -11,7 +11,9 @@ fn main() -> Result<(), Box> { let (nonce, sealed) = cipher.seal_random_to_vec(aad, plaintext)?; let opened = cipher.decrypt_to_vec(&nonce, aad, &sealed)?; - assert_eq!(opened, plaintext); + if opened != plaintext { + return Err(std::io::Error::other("AEAD round-trip changed the plaintext").into()); + } println!("ChaCha20-Poly1305 sealed {} bytes", plaintext.len()); Ok(()) } diff --git a/examples/basic.rs b/examples/basic.rs index b2ac356d..03c632cf 100644 --- a/examples/basic.rs +++ b/examples/basic.rs @@ -3,7 +3,7 @@ //! //! Run with: `cargo run --example basic --features full,getrandom` -use std::io::{Cursor, Read, Write}; +use std::io::{Read, Write}; use rscrypto::{ Aead, Blake3, ChaCha20Poly1305, ChaCha20Poly1305Key, Checksum, Crc32C, Digest, Ed25519Keypair, Ed25519SecretKey, @@ -86,17 +86,19 @@ fn auth_api() -> Result<(), Box> { let mut mac = HmacSha256::new(key); mac.update(b"hello "); mac.update(b"world"); - assert!(mac.verify(&tag).is_ok()); + mac.verify(&tag)?; mac.reset(); mac.update(data); - assert!(mac.verify(&tag).is_ok()); + mac.verify(&tag)?; let hkdf = HkdfSha256::new(b"salt", b"input key material"); let mut okm = [0u8; 42]; hkdf.expand(b"context", &mut okm)?; let oneshot = HkdfSha256::derive_array::<42>(b"salt", b"input key material", b"context")?; - assert_eq!(okm, oneshot); + if okm != oneshot { + return Err(std::io::Error::other("HKDF streaming and one-shot outputs differ").into()); + } println!("HMAC-SHA256 tag size = {} bytes", tag.as_slice().len()); println!("HKDF-SHA256 output = {} bytes\n", okm.len()); @@ -116,7 +118,9 @@ fn aead_api() -> Result<(), Box> { let mut opened = [0u8; 5]; aead.decrypt(&nonce, b"", &sealed, &mut opened)?; - assert_eq!(&opened, b"hello"); + if &opened != b"hello" { + return Err(std::io::Error::other("AEAD round-trip changed the plaintext").into()); + } println!("ChaCha20-Poly1305 round-trip succeeded"); println!(" nonce = {nonce}\n"); @@ -135,12 +139,16 @@ fn hex_api() -> Result<(), Box> { println!("Debug: {nonce:?}"); let parsed: Nonce96 = "abababababababababababab".parse()?; - assert_eq!(parsed, nonce); + if parsed != nonce { + return Err(std::io::Error::other("nonce hex round-trip changed the value").into()); + } println!("FromStr: round-trip succeeded"); let key = ChaCha20Poly1305Key::from_bytes([0x42; 32]); let key_debug = format!("{key:?}"); - assert_eq!(key_debug, "ChaCha20Poly1305Key(****)"); + if key_debug != "ChaCha20Poly1305Key(****)" { + return Err(std::io::Error::other("secret-key Debug output exposed an unexpected representation").into()); + } println!("\nSecret Debug: {key_debug}"); let ed_sk = Ed25519SecretKey::from_bytes([7u8; 32]); @@ -215,23 +223,35 @@ fn io_api() -> std::io::Result<()> { let data = b"stream me through adapters"; - let mut reader = Sha256::reader(Cursor::new(data.to_vec())); + let mut reader = Sha256::reader(data.as_slice()); let mut copied = Vec::new(); reader.read_to_end(&mut copied)?; - assert_eq!(copied, data); - assert_eq!(reader.digest(), Sha256::digest(data)); + if copied != data { + return Err(std::io::Error::other("digest reader changed the copied data")); + } + if reader.digest() != Sha256::digest(data) { + return Err(std::io::Error::other("digest reader produced the wrong digest")); + } let mut checksum_writer = Crc32C::writer(Vec::new()); checksum_writer.write_all(data)?; let (written, crc) = checksum_writer.into_parts(); - assert_eq!(written, data); - assert_eq!(crc, Crc32C::checksum(data)); + if written != data { + return Err(std::io::Error::other("checksum writer changed the written data")); + } + if crc != Crc32C::checksum(data) { + return Err(std::io::Error::other("checksum writer produced the wrong checksum")); + } let mut digest_writer = Blake3::writer(Vec::new()); digest_writer.write_all(data)?; let (written, digest) = digest_writer.into_parts(); - assert_eq!(written, data); - assert_eq!(digest, Blake3::digest(data)); + if written != data { + return Err(std::io::Error::other("digest writer changed the written data")); + } + if digest != Blake3::digest(data) { + return Err(std::io::Error::other("digest writer produced the wrong digest")); + } println!("reader digest matches Sha256::digest()"); println!("writer checksum matches Crc32C::checksum()"); diff --git a/examples/mlkem_encapsulation.rs b/examples/mlkem_encapsulation.rs index 83721d4d..577a650c 100644 --- a/examples/mlkem_encapsulation.rs +++ b/examples/mlkem_encapsulation.rs @@ -7,7 +7,9 @@ fn main() -> Result<(), Box> { let (ciphertext, shared_secret) = MlKem768::try_encapsulate(&encapsulation_key)?; let decapsulated = MlKem768::decapsulate(&decapsulation_key, &ciphertext)?; - assert!(shared_secret.ct_eq(&decapsulated).declassify()); + if !shared_secret.ct_eq(&decapsulated).declassify() { + return Err(std::io::Error::other("ML-KEM encapsulation and decapsulation secrets differ").into()); + } println!( "ML-KEM-768 encapsulated {} shared-secret bytes", shared_secret.as_bytes().len() diff --git a/examples/parallel.rs b/examples/parallel.rs index 8f924712..f66d8aaa 100644 --- a/examples/parallel.rs +++ b/examples/parallel.rs @@ -63,7 +63,7 @@ fn parallel_chunks() { println!("Parallel chunk processing\n"); // Simulate large data (in practice, this could be a memory-mapped file) - let data: Vec = (0..1_000_000).map(|i| (i % 256) as u8).collect(); + let data: Vec = (0usize..1_000_000).map(|i| i.to_le_bytes()[0]).collect(); let chunk_size = 250_000; // 4 chunks of 250KB each @@ -74,12 +74,9 @@ fn parallel_chunks() { // Parallel: compute each chunk's CRC, then combine let chunks: Vec<_> = data.chunks(chunk_size).collect(); let chunk_crcs: Vec<_> = thread::scope(|scope| { - let handles: Vec<_> = chunks + chunks .iter() .map(|&chunk| scope.spawn(move || Crc64::checksum(chunk))) - .collect(); - handles - .into_iter() .map(|handle| handle.join().expect("thread panicked")) .collect() }); @@ -100,7 +97,12 @@ fn threaded_example() { println!("Threaded processing\n"); // Generate test data - let data: Vec = (0..4_000_000).map(|i| ((i * 17) % 256) as u8).collect(); + let data: Vec = (0usize..4_000_000) + .map(|i| { + let value = i.strict_mul(17).strict_rem(256); + u8::try_from(value).expect("threaded fixture byte must fit in u8") + }) + .collect(); let num_threads = 4; let chunk_size = data.len() / num_threads; @@ -114,12 +116,9 @@ fn threaded_example() { // Spawn scoped threads to compute each chunk's CRC without copying chunk data. let mut results: Vec<(usize, u64, usize)> = thread::scope(|scope| { - let handles: Vec<_> = chunks + chunks .iter() .map(|&(idx, chunk)| scope.spawn(move || (idx, Crc64::checksum(chunk), chunk.len()))) - .collect(); - handles - .into_iter() .map(|handle| handle.join().expect("thread panicked")) .collect() }); diff --git a/examples/password_hashing.rs b/examples/password_hashing.rs index 071cb91d..a93ad8cd 100644 --- a/examples/password_hashing.rs +++ b/examples/password_hashing.rs @@ -13,13 +13,17 @@ fn main() -> Result<(), Box> { let argon2 = Argon2idPassword::default(); let argon2_phc = argon2.hash_password(password)?; - assert!(argon2.verify_password(password, &argon2_phc).is_ok()); - assert!(argon2.verify_password(b"wrong password", &argon2_phc).is_err()); + argon2.verify_password(password, &argon2_phc)?; + if argon2.verify_password(b"wrong password", &argon2_phc).is_ok() { + return Err(std::io::Error::other("Argon2id accepted the wrong password").into()); + } let scrypt = ScryptPassword::default(); let scrypt_phc = scrypt.hash_password(password)?; - assert!(scrypt.verify_password(password, &scrypt_phc).is_ok()); - assert!(scrypt.verify_password(b"wrong password", &scrypt_phc).is_err()); + scrypt.verify_password(password, &scrypt_phc)?; + if scrypt.verify_password(b"wrong password", &scrypt_phc).is_ok() { + return Err(std::io::Error::other("scrypt accepted the wrong password").into()); + } println!("{argon2_phc}"); println!("{scrypt_phc}"); diff --git a/fuzz-packages/aead-aegis256/Cargo.lock b/fuzz-packages/aead-aegis256/Cargo.lock index 3a480232..309d2ad4 100644 --- a/fuzz-packages/aead-aegis256/Cargo.lock +++ b/fuzz-packages/aead-aegis256/Cargo.lock @@ -20,9 +20,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -38,9 +38,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -87,7 +87,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-aegis256" diff --git a/fuzz-packages/aead-aes128gcm/Cargo.lock b/fuzz-packages/aead-aes128gcm/Cargo.lock index 520c1bcb..eb550e78 100644 --- a/fuzz-packages/aead-aes128gcm/Cargo.lock +++ b/fuzz-packages/aead-aes128gcm/Cargo.lock @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -218,7 +218,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-aes128gcm" diff --git a/fuzz-packages/aead-aes128gcmsiv/Cargo.lock b/fuzz-packages/aead-aes128gcmsiv/Cargo.lock index 7e8bc8ba..d374eba4 100644 --- a/fuzz-packages/aead-aes128gcmsiv/Cargo.lock +++ b/fuzz-packages/aead-aes128gcmsiv/Cargo.lock @@ -4,30 +4,30 @@ version = 4 [[package]] name = "aead" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ "crypto-common", - "generic-array", + "inout", ] [[package]] name = "aes" -version = "0.8.4" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" dependencies = [ - "cfg-if", "cipher", + "cpubits", "cpufeatures", ] [[package]] name = "aes-gcm-siv" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0784134ba9375416d469ec31e7c5f9fa94405049cf08c5ce5b4698be673e0d" +checksum = "1f437e5b075722bda3f54039e95d60d1c142f140e6c45b718e4f9fca3e5a1514" dependencies = [ "aead", "aes", @@ -35,7 +35,6 @@ dependencies = [ "ctr", "polyval", "subtle", - "zeroize", ] [[package]] @@ -44,11 +43,20 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -64,57 +72,68 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cipher" -version = "0.4.4" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ + "block-buffer", "crypto-common", "inout", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", ] [[package]] name = "ctr" -version = "0.9.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" dependencies = [ "cipher", ] [[package]] -name = "find-msvc-tools" -version = "0.1.9" +name = "ctutils" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] [[package]] -name = "generic-array" -version = "0.14.7" +name = "find-msvc-tools" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -127,13 +146,22 @@ dependencies = [ "r-efi", ] +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "inout" -version = "0.1.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -162,21 +190,14 @@ dependencies = [ "cc", ] -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - [[package]] name = "polyval" -version = "0.6.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ - "cfg-if", + "cpubits", "cpufeatures", - "opaque-debug", "universal-hash", ] @@ -188,7 +209,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-aes128gcmsiv" @@ -227,22 +248,10 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "universal-hash" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" dependencies = [ "crypto-common", - "subtle", + "ctutils", ] - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "zeroize" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" diff --git a/fuzz-packages/aead-aes128gcmsiv/Cargo.toml b/fuzz-packages/aead-aes128gcmsiv/Cargo.toml index 2f86b491..e9b6ce9d 100644 --- a/fuzz-packages/aead-aes128gcmsiv/Cargo.toml +++ b/fuzz-packages/aead-aes128gcmsiv/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" rscrypto = { path = "../..", features = ["std", "aes-gcm-siv"] } rscrypto_fuzz = { package = "rscrypto-fuzz-support", path = "../../fuzz/support", features = ["aead"] } -aes-gcm-siv = { version = "0.11", default-features = false, features = ["alloc", "aes"] } +aes-gcm-siv = { version = "0.12", default-features = false, features = ["alloc", "aes"] } [workspace] members = ["."] diff --git a/fuzz-packages/aead-aes256gcm/Cargo.lock b/fuzz-packages/aead-aes256gcm/Cargo.lock index 5ef7f9e0..a94e9b6d 100644 --- a/fuzz-packages/aead-aes256gcm/Cargo.lock +++ b/fuzz-packages/aead-aes256gcm/Cargo.lock @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -218,7 +218,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-aes256gcm" diff --git a/fuzz-packages/aead-aes256gcmsiv/Cargo.lock b/fuzz-packages/aead-aes256gcmsiv/Cargo.lock index dfa5d6a3..0a87b8b3 100644 --- a/fuzz-packages/aead-aes256gcmsiv/Cargo.lock +++ b/fuzz-packages/aead-aes256gcmsiv/Cargo.lock @@ -4,30 +4,30 @@ version = 4 [[package]] name = "aead" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ "crypto-common", - "generic-array", + "inout", ] [[package]] name = "aes" -version = "0.8.4" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" dependencies = [ - "cfg-if", "cipher", + "cpubits", "cpufeatures", ] [[package]] name = "aes-gcm-siv" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0784134ba9375416d469ec31e7c5f9fa94405049cf08c5ce5b4698be673e0d" +checksum = "1f437e5b075722bda3f54039e95d60d1c142f140e6c45b718e4f9fca3e5a1514" dependencies = [ "aead", "aes", @@ -35,7 +35,6 @@ dependencies = [ "ctr", "polyval", "subtle", - "zeroize", ] [[package]] @@ -44,11 +43,20 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -64,57 +72,68 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cipher" -version = "0.4.4" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ + "block-buffer", "crypto-common", "inout", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", ] [[package]] name = "ctr" -version = "0.9.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" dependencies = [ "cipher", ] [[package]] -name = "find-msvc-tools" -version = "0.1.9" +name = "ctutils" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] [[package]] -name = "generic-array" -version = "0.14.7" +name = "find-msvc-tools" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -127,13 +146,22 @@ dependencies = [ "r-efi", ] +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "inout" -version = "0.1.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -162,21 +190,14 @@ dependencies = [ "cc", ] -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - [[package]] name = "polyval" -version = "0.6.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ - "cfg-if", + "cpubits", "cpufeatures", - "opaque-debug", "universal-hash", ] @@ -188,7 +209,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-aes256gcmsiv" @@ -227,22 +248,10 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "universal-hash" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" dependencies = [ "crypto-common", - "subtle", + "ctutils", ] - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "zeroize" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" diff --git a/fuzz-packages/aead-aes256gcmsiv/Cargo.toml b/fuzz-packages/aead-aes256gcmsiv/Cargo.toml index 7c2a09e6..aa480bc9 100644 --- a/fuzz-packages/aead-aes256gcmsiv/Cargo.toml +++ b/fuzz-packages/aead-aes256gcmsiv/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" rscrypto = { path = "../..", features = ["std", "aes-gcm-siv"] } rscrypto_fuzz = { package = "rscrypto-fuzz-support", path = "../../fuzz/support", features = ["aead"] } -aes-gcm-siv = { version = "0.11", default-features = false, features = ["alloc", "aes"] } +aes-gcm-siv = { version = "0.12", default-features = false, features = ["alloc", "aes"] } [workspace] members = ["."] diff --git a/fuzz-packages/aead-ascon128/Cargo.lock b/fuzz-packages/aead-ascon128/Cargo.lock index 15bc01d0..575ddea4 100644 --- a/fuzz-packages/aead-ascon128/Cargo.lock +++ b/fuzz-packages/aead-ascon128/Cargo.lock @@ -38,9 +38,9 @@ checksum = "c90c8493a0b3bf8678a82f096821a2c527dd6344bbcef6265bfcc09745f44e24" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -65,9 +65,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -132,7 +132,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-ascon128" diff --git a/fuzz-packages/aead-chacha20poly1305/Cargo.lock b/fuzz-packages/aead-chacha20poly1305/Cargo.lock index cd2ff2a1..fbfc0c17 100644 --- a/fuzz-packages/aead-chacha20poly1305/Cargo.lock +++ b/fuzz-packages/aead-chacha20poly1305/Cargo.lock @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -114,9 +114,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -191,7 +191,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-chacha20poly1305" diff --git a/fuzz-packages/aead-nonce-counter/Cargo.lock b/fuzz-packages/aead-nonce-counter/Cargo.lock index 5b5f25ce..e05e16db 100644 --- a/fuzz-packages/aead-nonce-counter/Cargo.lock +++ b/fuzz-packages/aead-nonce-counter/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,9 +28,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-nonce-counter" diff --git a/fuzz-packages/aead-xchacha20poly1305/Cargo.lock b/fuzz-packages/aead-xchacha20poly1305/Cargo.lock index 3b19f3f1..33393cc2 100644 --- a/fuzz-packages/aead-xchacha20poly1305/Cargo.lock +++ b/fuzz-packages/aead-xchacha20poly1305/Cargo.lock @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -114,9 +114,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -191,7 +191,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-xchacha20poly1305" diff --git a/fuzz-packages/auth-argon2/Cargo.lock b/fuzz-packages/auth-argon2/Cargo.lock index 8fb4467c..778af033 100644 --- a/fuzz-packages/auth-argon2/Cargo.lock +++ b/fuzz-packages/auth-argon2/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "argon2" -version = "0.6.0-rc.8" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7af50940b73bf4e16c15c448a2b121c63f2d68e3e54b6a8731673cb4aa0cdff5" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", "blake2", @@ -28,27 +28,27 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "blake2" -version = "0.11.0-rc.6" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061f1a09225e328e1ffbb378d2d49923c0ca5fee19fb5ac1cc9c1e9d52b93690" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ "digest", ] [[package]] name = "block-buffer" -version = "0.12.1" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "hybrid-array", + "generic-array", ] [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -62,55 +62,51 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "cmov" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" - [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] [[package]] name = "crypto-common" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" -dependencies = [ - "hybrid-array", -] - -[[package]] -name = "ctutils" -version = "0.4.2" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "cmov", + "generic-array", + "typenum", ] [[package]] name = "digest" -version = "0.11.3" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "ctutils", + "subtle", ] [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] [[package]] name = "getrandom" @@ -123,15 +119,6 @@ dependencies = [ "r-efi", ] -[[package]] -name = "hybrid-array" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" -dependencies = [ - "typenum", -] - [[package]] name = "jobserver" version = "0.1.35" @@ -160,21 +147,13 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" -dependencies = [ - "phc", -] - -[[package]] -name = "phc" -version = "0.6.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ "base64ct", - "ctutils", + "rand_core", + "subtle", ] [[package]] @@ -183,9 +162,15 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-argon2" @@ -210,8 +195,20 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" diff --git a/fuzz-packages/auth-argon2/Cargo.toml b/fuzz-packages/auth-argon2/Cargo.toml index 979d119e..5c417308 100644 --- a/fuzz-packages/auth-argon2/Cargo.toml +++ b/fuzz-packages/auth-argon2/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" rscrypto = { path = "../..", features = ["std", "argon2"] } rscrypto_fuzz = { package = "rscrypto-fuzz-support", path = "../../fuzz/support" } -argon2 = { version = "0.6.0-rc.8", default-features = false, features = ["alloc"] } +argon2 = { version = "0.5.3", default-features = false, features = ["alloc"] } [workspace] members = ["."] diff --git a/fuzz-packages/auth-ed25519/Cargo.lock b/fuzz-packages/auth-ed25519/Cargo.lock index 013bb2aa..12386b92 100644 --- a/fuzz-packages/auth-ed25519/Cargo.lock +++ b/fuzz-packages/auth-ed25519/Cargo.lock @@ -8,26 +8,20 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -[[package]] -name = "base64ct" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" - [[package]] name = "block-buffer" -version = "0.10.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -41,36 +35,29 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", ] [[package]] name = "curve25519-dalek" -version = "4.1.3" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" dependencies = [ "cfg-if", "cpufeatures", @@ -90,24 +77,14 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", -] - -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid", - "zeroize", + "syn", ] [[package]] name = "digest" -version = "0.10.7" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer", "crypto-common", @@ -115,23 +92,21 @@ dependencies = [ [[package]] name = "ed25519" -version = "2.2.3" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" dependencies = [ - "pkcs8", "signature", ] [[package]] name = "ed25519-dalek" -version = "2.2.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" dependencies = [ "curve25519-dalek", "ed25519", - "serde", "sha2", "subtle", "zeroize", @@ -139,46 +114,34 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.2.9" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" [[package]] name = "find-msvc-tools" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - -[[package]] -name = "generic-array" -version = "0.14.7" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" -version = "0.2.17" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "wasi", + "r-efi", ] [[package]] -name = "getrandom" -version = "0.4.3" +name = "hybrid-array" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ - "cfg-if", - "libc", - "r-efi", + "typenum", ] [[package]] @@ -187,7 +150,7 @@ version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.4.3", + "getrandom", "libc", ] @@ -207,16 +170,6 @@ dependencies = [ "cc", ] -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - [[package]] name = "proc-macro2" version = "1.0.107" @@ -241,18 +194,9 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-ed25519" @@ -286,40 +230,11 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", "cpufeatures", @@ -334,22 +249,9 @@ checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signature" -version = "2.2.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "rand_core", -] - -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" [[package]] name = "subtle" @@ -368,17 +270,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "typenum" version = "1.20.1" @@ -391,18 +282,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - [[package]] name = "zeroize" version = "1.9.0" diff --git a/fuzz-packages/auth-ed25519/Cargo.toml b/fuzz-packages/auth-ed25519/Cargo.toml index a56bbf2c..3dcfbad7 100644 --- a/fuzz-packages/auth-ed25519/Cargo.toml +++ b/fuzz-packages/auth-ed25519/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" rscrypto = { path = "../..", features = ["std", "ed25519"] } rscrypto_fuzz = { package = "rscrypto-fuzz-support", path = "../../fuzz/support" } -ed25519-dalek = "2.2" +ed25519-dalek = "3.0" [workspace] members = ["."] diff --git a/fuzz-packages/auth-hkdf/Cargo.lock b/fuzz-packages/auth-hkdf/Cargo.lock index c4719c32..88526fb3 100644 --- a/fuzz-packages/auth-hkdf/Cargo.lock +++ b/fuzz-packages/auth-hkdf/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -88,9 +88,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -164,7 +164,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-hkdf" diff --git a/fuzz-packages/auth-hmac/Cargo.lock b/fuzz-packages/auth-hmac/Cargo.lock index e14dfe58..f9453d1f 100644 --- a/fuzz-packages/auth-hmac/Cargo.lock +++ b/fuzz-packages/auth-hmac/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -88,9 +88,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -155,7 +155,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-hmac" diff --git a/fuzz-packages/auth-kmac256/Cargo.lock b/fuzz-packages/auth-kmac256/Cargo.lock index ed5bce65..3ecd0c71 100644 --- a/fuzz-packages/auth-kmac256/Cargo.lock +++ b/fuzz-packages/auth-kmac256/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -34,9 +34,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -83,7 +83,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-kmac256" diff --git a/fuzz-packages/auth-pbkdf2/Cargo.lock b/fuzz-packages/auth-pbkdf2/Cargo.lock index a661e37c..ea2872ba 100644 --- a/fuzz-packages/auth-pbkdf2/Cargo.lock +++ b/fuzz-packages/auth-pbkdf2/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -88,9 +88,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -165,7 +165,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-pbkdf2" diff --git a/fuzz-packages/auth-phc/Cargo.lock b/fuzz-packages/auth-phc/Cargo.lock index 0aac3f41..39a50a0e 100644 --- a/fuzz-packages/auth-phc/Cargo.lock +++ b/fuzz-packages/auth-phc/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,9 +28,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-phc" diff --git a/fuzz-packages/auth-scrypt/Cargo.lock b/fuzz-packages/auth-scrypt/Cargo.lock index 5a7deb35..f889a2a5 100644 --- a/fuzz-packages/auth-scrypt/Cargo.lock +++ b/fuzz-packages/auth-scrypt/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -92,9 +92,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -178,7 +178,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-scrypt" diff --git a/fuzz-packages/auth-x25519/Cargo.lock b/fuzz-packages/auth-x25519/Cargo.lock index 4267b661..c8a42896 100644 --- a/fuzz-packages/auth-x25519/Cargo.lock +++ b/fuzz-packages/auth-x25519/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,18 +28,18 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] [[package]] name = "curve25519-dalek" -version = "4.1.3" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" dependencies = [ "cfg-if", "cpufeatures", @@ -58,20 +58,20 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "fiat-crypto" -version = "0.2.9" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -136,13 +136,13 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand_core" -version = "0.6.4" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-x25519" @@ -176,36 +176,6 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - [[package]] name = "shlex" version = "2.0.1" @@ -229,17 +199,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "unicode-ident" version = "1.0.24" @@ -248,13 +207,12 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "x25519-dalek" -version = "2.0.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +checksum = "e7e8131a03190127fb2263afc72b322ecadae46b6ff8c6f399ff5d02f5559af6" dependencies = [ "curve25519-dalek", "rand_core", - "serde", "zeroize", ] @@ -263,17 +221,3 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] diff --git a/fuzz-packages/auth-x25519/Cargo.toml b/fuzz-packages/auth-x25519/Cargo.toml index eb7ec9b6..904467b1 100644 --- a/fuzz-packages/auth-x25519/Cargo.toml +++ b/fuzz-packages/auth-x25519/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" rscrypto = { path = "../..", features = ["std", "x25519"] } rscrypto_fuzz = { package = "rscrypto-fuzz-support", path = "../../fuzz/support" } -x25519-dalek = { version = "2.0.1", features = ["static_secrets"] } +x25519-dalek = { version = "3.0.0", features = ["static_secrets"] } [workspace] members = ["."] diff --git a/fuzz-packages/checksums-crc16/Cargo.lock b/fuzz-packages/checksums-crc16/Cargo.lock index 1c1a2e88..6c59c848 100644 --- a/fuzz-packages/checksums-crc16/Cargo.lock +++ b/fuzz-packages/checksums-crc16/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -43,9 +43,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -92,7 +92,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-checksums-crc16" diff --git a/fuzz-packages/checksums-crc24/Cargo.lock b/fuzz-packages/checksums-crc24/Cargo.lock index b8a0bfa6..043526cd 100644 --- a/fuzz-packages/checksums-crc24/Cargo.lock +++ b/fuzz-packages/checksums-crc24/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -43,9 +43,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -92,7 +92,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-checksums-crc24" diff --git a/fuzz-packages/checksums-crc32/Cargo.lock b/fuzz-packages/checksums-crc32/Cargo.lock index 76ed2839..eb4945bb 100644 --- a/fuzz-packages/checksums-crc32/Cargo.lock +++ b/fuzz-packages/checksums-crc32/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -43,9 +43,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -92,7 +92,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-checksums-crc32" diff --git a/fuzz-packages/checksums-crc64/Cargo.lock b/fuzz-packages/checksums-crc64/Cargo.lock index c910382d..d9b51307 100644 --- a/fuzz-packages/checksums-crc64/Cargo.lock +++ b/fuzz-packages/checksums-crc64/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -43,9 +43,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -92,7 +92,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-checksums-crc64" diff --git a/fuzz-packages/fast-rapidhash/Cargo.lock b/fuzz-packages/fast-rapidhash/Cargo.lock index 0f99f2e3..6026a714 100644 --- a/fuzz-packages/fast-rapidhash/Cargo.lock +++ b/fuzz-packages/fast-rapidhash/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,9 +28,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -86,7 +86,7 @@ dependencies = [ [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-fast-rapidhash" diff --git a/fuzz-packages/fast-xxh3/Cargo.lock b/fuzz-packages/fast-xxh3/Cargo.lock index 207a69f6..640364b1 100644 --- a/fuzz-packages/fast-xxh3/Cargo.lock +++ b/fuzz-packages/fast-xxh3/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,9 +28,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-fast-xxh3" diff --git a/fuzz-packages/hash-ascon/Cargo.lock b/fuzz-packages/hash-ascon/Cargo.lock index 946b3949..c849763e 100644 --- a/fuzz-packages/hash-ascon/Cargo.lock +++ b/fuzz-packages/hash-ascon/Cargo.lock @@ -35,9 +35,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -72,9 +72,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -130,7 +130,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-hash-ascon" diff --git a/fuzz-packages/hash-blake2/Cargo.lock b/fuzz-packages/hash-blake2/Cargo.lock index d993c23c..c2567352 100644 --- a/fuzz-packages/hash-blake2/Cargo.lock +++ b/fuzz-packages/hash-blake2/Cargo.lock @@ -10,11 +10,20 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "blake2" -version = "0.11.0-rc.6" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061f1a09225e328e1ffbb378d2d49923c0ca5fee19fb5ac1cc9c1e9d52b93690" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", ] [[package]] @@ -28,9 +37,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -50,6 +59,16 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -68,22 +87,43 @@ dependencies = [ "cmov", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.12.1", + "crypto-common 0.2.2", "ctutils", ] [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] [[package]] name = "getrandom" @@ -102,7 +142,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -148,14 +188,14 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-hash-blake2" version = "0.0.0" dependencies = [ "blake2", - "digest", + "digest 0.11.3", "hmac", "libfuzzer-sys", "rscrypto", @@ -175,8 +215,20 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" diff --git a/fuzz-packages/hash-blake2/Cargo.toml b/fuzz-packages/hash-blake2/Cargo.toml index e9db192f..b9424e7b 100644 --- a/fuzz-packages/hash-blake2/Cargo.toml +++ b/fuzz-packages/hash-blake2/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" rscrypto = { path = "../..", features = ["std", "blake2b", "blake2s"] } rscrypto_fuzz = { package = "rscrypto-fuzz-support", path = "../../fuzz/support" } -blake2 = "0.11.0-rc.6" +blake2 = "0.10.6" digest = "0.11" hmac = "0.13" diff --git a/fuzz-packages/hash-blake3/Cargo.lock b/fuzz-packages/hash-blake3/Cargo.lock index 61858c38..fd940359 100644 --- a/fuzz-packages/hash-blake3/Cargo.lock +++ b/fuzz-packages/hash-blake3/Cargo.lock @@ -22,9 +22,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" dependencies = [ "arrayref", "arrayvec", @@ -36,9 +36,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -69,9 +69,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -118,7 +118,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-hash-blake3" diff --git a/fuzz-packages/hash-sha2/Cargo.lock b/fuzz-packages/hash-sha2/Cargo.lock index b66776e7..3a7a8f3e 100644 --- a/fuzz-packages/hash-sha2/Cargo.lock +++ b/fuzz-packages/hash-sha2/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -72,9 +72,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -130,7 +130,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-hash-sha2" diff --git a/fuzz-packages/hash-sha3/Cargo.lock b/fuzz-packages/hash-sha3/Cargo.lock index 23941a0e..bdfb4385 100644 --- a/fuzz-packages/hash-sha3/Cargo.lock +++ b/fuzz-packages/hash-sha3/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -89,9 +89,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "keccak" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +checksum = "ffd9697dc4a9a62e2da93389f34400b77a28f0287711263cabb203b3ccb9c0e4" dependencies = [ "cfg-if", "cpufeatures", @@ -157,7 +157,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-hash-sha3" diff --git a/fuzz-packages/surface-hex-parse/Cargo.lock b/fuzz-packages/surface-hex-parse/Cargo.lock index 722ecdf9..a6214cf4 100644 --- a/fuzz-packages/surface-hex-parse/Cargo.lock +++ b/fuzz-packages/surface-hex-parse/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,9 +28,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-support" diff --git a/fuzz-packages/traits-io/Cargo.lock b/fuzz-packages/traits-io/Cargo.lock index 0a4d28bb..27ef473c 100644 --- a/fuzz-packages/traits-io/Cargo.lock +++ b/fuzz-packages/traits-io/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,9 +28,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-support" diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 73ccdeee..b438ed0e 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -2,16 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "aead" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" -dependencies = [ - "crypto-common 0.1.7", - "generic-array", -] - [[package]] name = "aead" version = "0.6.1" @@ -19,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ "crypto-common 0.2.2", - "inout 0.2.2", + "inout", ] [[package]] @@ -32,24 +22,13 @@ dependencies = [ "softaes", ] -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher 0.4.4", - "cpufeatures 0.2.17", -] - [[package]] name = "aes" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" dependencies = [ - "cipher 0.5.2", + "cipher", "cpubits", "cpufeatures 0.3.0", ] @@ -60,27 +39,26 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ - "aead 0.6.1", - "aes 0.9.2", - "cipher 0.5.2", - "ctr 0.10.1", + "aead", + "aes", + "cipher", + "ctr", "ghash", "subtle", ] [[package]] name = "aes-gcm-siv" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0784134ba9375416d469ec31e7c5f9fa94405049cf08c5ce5b4698be673e0d" +checksum = "1f437e5b075722bda3f54039e95d60d1c142f140e6c45b718e4f9fca3e5a1514" dependencies = [ - "aead 0.5.2", - "aes 0.8.4", - "cipher 0.4.4", - "ctr 0.9.2", - "polyval 0.6.2", + "aead", + "aes", + "cipher", + "ctr", + "polyval", "subtle", - "zeroize", ] [[package]] @@ -91,13 +69,13 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "argon2" -version = "0.6.0-rc.8" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7af50940b73bf4e16c15c448a2b121c63f2d68e3e54b6a8731673cb4aa0cdff5" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", "blake2", - "cpufeatures 0.3.0", + "cpufeatures 0.2.17", "password-hash", ] @@ -119,9 +97,9 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6cba922132fed390c20be89a16fa713999c46b2ce84ac4b0ecee0f3586a39d88" dependencies = [ - "aead 0.6.1", + "aead", "ascon-core", - "inout 0.2.2", + "inout", "subtle", ] @@ -147,12 +125,6 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - [[package]] name = "base16ct" version = "1.0.0" @@ -167,18 +139,18 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "blake2" -version = "0.11.0-rc.6" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061f1a09225e328e1ffbb378d2d49923c0ca5fee19fb5ac1cc9c1e9d52b93690" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest 0.11.3", + "digest 0.10.7", ] [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" dependencies = [ "arrayref", "arrayvec", @@ -208,9 +180,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -231,7 +203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cipher 0.5.2", + "cipher", "cpufeatures 0.3.0", ] @@ -241,22 +213,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b89e1c441e926b9c82a8d023f6e1b7ae0adcfaa7d621814e4d60789bac751cb" dependencies = [ - "aead 0.6.1", + "aead", "chacha20", - "cipher 0.5.2", + "cipher", "poly1305", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common 0.1.7", - "inout 0.1.4", -] - [[package]] name = "cipher" version = "0.5.2" @@ -265,7 +227,7 @@ checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ "block-buffer 0.12.1", "crypto-common 0.2.2", - "inout 0.2.2", + "inout", ] [[package]] @@ -274,12 +236,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "const-oid" version = "0.10.2" @@ -337,18 +293,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" -[[package]] -name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - [[package]] name = "crypto-bigint" version = "0.7.5" @@ -357,7 +301,7 @@ checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" dependencies = [ "cpubits", "ctutils", - "getrandom 0.4.3", + "getrandom", "hybrid-array", "num-traits", "rand_core 0.10.1", @@ -381,7 +325,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "getrandom 0.4.3", + "getrandom", "hybrid-array", "rand_core 0.10.1", ] @@ -397,22 +341,13 @@ dependencies = [ "sponge-cursor", ] -[[package]] -name = "ctr" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" -dependencies = [ - "cipher 0.4.4", -] - [[package]] name = "ctr" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" dependencies = [ - "cipher 0.5.2", + "cipher", ] [[package]] @@ -427,14 +362,14 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.3" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures 0.3.0", "curve25519-dalek-derive", - "digest 0.10.7", + "digest 0.11.3", "fiat-crypto", "rustc_version", "subtle", @@ -452,23 +387,13 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid 0.9.6", - "zeroize", -] - [[package]] name = "der" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" dependencies = [ - "const-oid 0.10.2", + "const-oid", "zeroize", ] @@ -479,7 +404,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid 0.9.6", "crypto-common 0.1.7", "subtle", ] @@ -491,79 +415,44 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", - "const-oid 0.10.2", + "const-oid", "crypto-common 0.2.2", "ctutils", ] -[[package]] -name = "ecdsa" -version = "0.16.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der 0.7.10", - "digest 0.10.7", - "elliptic-curve 0.13.8", - "rfc6979 0.4.0", - "signature 2.2.0", - "spki 0.7.3", -] - [[package]] name = "ecdsa" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ - "der 0.8.1", + "der", "digest 0.11.3", - "elliptic-curve 0.14.1", - "rfc6979 0.6.0", - "signature 3.0.0", - "spki 0.8.0", + "elliptic-curve", + "rfc6979", + "signature", + "spki", "zeroize", ] [[package]] name = "ed25519" -version = "2.2.3" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" dependencies = [ - "pkcs8 0.10.2", - "signature 2.2.0", + "signature", ] [[package]] name = "ed25519-dalek" -version = "2.2.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" dependencies = [ "curve25519-dalek", "ed25519", - "serde", - "sha2 0.10.9", - "subtle", - "zeroize", -] - -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct 0.2.0", - "crypto-bigint 0.5.5", - "digest 0.10.7", - "ff 0.13.1", - "generic-array", - "group 0.13.0", - "pkcs8 0.10.2", - "rand_core 0.6.4", - "sec1 0.7.3", + "sha2", "subtle", "zeroize", ] @@ -574,30 +463,20 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" dependencies = [ - "base16ct 1.0.0", - "crypto-bigint 0.7.5", + "base16ct", + "crypto-bigint", "crypto-common 0.2.2", "digest 0.11.3", - "ff 0.14.0", - "group 0.14.0", + "ff", + "group", "hybrid-array", - "pkcs8 0.11.0", + "pkcs8", "rand_core 0.10.1", - "sec1 0.8.1", + "sec1", "subtle", "zeroize", ] -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "ff" version = "0.14.0" @@ -610,15 +489,15 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.2.9" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "generic-array" @@ -628,18 +507,6 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", - "zeroize", -] - -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "libc", - "wasi", ] [[package]] @@ -660,18 +527,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ - "polyval 0.7.3", -] - -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff 0.13.1", - "rand_core 0.6.4", - "subtle", + "polyval", ] [[package]] @@ -680,7 +536,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" dependencies = [ - "ff 0.14.0", + "ff", "rand_core 0.10.1", "subtle", ] @@ -691,16 +547,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" dependencies = [ - "hmac 0.13.0", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", + "hmac", ] [[package]] @@ -723,15 +570,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] - [[package]] name = "inout" version = "0.2.2" @@ -747,15 +585,15 @@ version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.4.3", + "getrandom", "libc", ] [[package]] name = "keccak" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +checksum = "ffd9697dc4a9a62e2da93389f34400b77a28f0287711263cabb203b3ccb9c0e4" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -792,44 +630,42 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - [[package]] name = "p256" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a" dependencies = [ - "ecdsa 0.17.0", - "elliptic-curve 0.14.1", + "ecdsa", + "elliptic-curve", "primefield", - "primeorder 0.14.0", - "sha2 0.11.0", + "primeorder", + "sha2", ] [[package]] name = "p384" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +checksum = "d17b851e6b3e378ab4ecb07fa2ed23f4d15f075735f8fec9fa1e7bdce5f8301f" dependencies = [ - "ecdsa 0.16.9", - "elliptic-curve 0.13.8", - "primeorder 0.13.6", - "sha2 0.10.9", + "ecdsa", + "elliptic-curve", + "fiat-crypto", + "primefield", + "primeorder", + "sha2", ] [[package]] name = "password-hash" -version = "0.6.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ - "phc", + "base64ct", + "rand_core 0.6.4", + "subtle", ] [[package]] @@ -839,27 +675,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ "digest 0.11.3", - "hmac 0.13.0", -] - -[[package]] -name = "phc" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" -dependencies = [ - "base64ct", - "ctutils", -] - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der 0.7.10", - "spki 0.7.3", + "hmac", ] [[package]] @@ -868,8 +684,8 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ - "der 0.8.1", - "spki 0.8.0", + "der", + "spki", ] [[package]] @@ -879,19 +695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2d0073b297041425c7c3df6eb4792d598a15323fe63346852b092eca02904c" dependencies = [ "cpufeatures 0.3.0", - "universal-hash 0.6.1", -] - -[[package]] -name = "polyval" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "opaque-debug", - "universal-hash 0.5.1", + "universal-hash", ] [[package]] @@ -902,7 +706,7 @@ checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ "cpubits", "cpufeatures 0.3.0", - "universal-hash 0.6.1", + "universal-hash", ] [[package]] @@ -911,30 +715,21 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" dependencies = [ - "crypto-bigint 0.7.5", + "crypto-bigint", "crypto-common 0.2.2", - "ff 0.14.0", + "ff", "rand_core 0.10.1", "subtle", "zeroize", ] -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve 0.13.8", -] - [[package]] name = "primeorder" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" dependencies = [ - "elliptic-curve 0.14.1", + "elliptic-curve", "once_cell", "primefield", "serdect", @@ -970,9 +765,6 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] [[package]] name = "rand_core" @@ -989,29 +781,19 @@ dependencies = [ "rustversion", ] -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac 0.12.1", - "subtle", -] - [[package]] name = "rfc6979" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" dependencies = [ - "crypto-bigint 0.7.5", - "hmac 0.13.0", + "crypto-bigint", + "hmac", ] [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz" @@ -1031,7 +813,7 @@ dependencies = [ "digest 0.11.3", "ed25519-dalek", "hkdf", - "hmac 0.13.0", + "hmac", "libfuzzer-sys", "p256", "p384", @@ -1040,7 +822,7 @@ dependencies = [ "rscrypto", "rscrypto-fuzz-support", "scrypt", - "sha2 0.11.0", + "sha2", "sha3", "tiny-keccak", "x25519-dalek", @@ -1076,7 +858,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f874456e72520ff1375a06c588eaf074b0f01f9e9e1aada45bd9b7954a6e42c" dependencies = [ "cfg-if", - "cipher 0.5.2", + "cipher", ] [[package]] @@ -1088,21 +870,7 @@ dependencies = [ "cfg-if", "pbkdf2", "salsa20", - "sha2 0.11.0", -] - -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct 0.2.0", - "der 0.7.10", - "generic-array", - "pkcs8 0.10.2", - "subtle", - "zeroize", + "sha2", ] [[package]] @@ -1111,9 +879,9 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" dependencies = [ - "base16ct 1.0.0", + "base16ct", "ctutils", - "der 0.8.1", + "der", "hybrid-array", "subtle", "zeroize", @@ -1132,7 +900,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", - "serde_derive", ] [[package]] @@ -1161,21 +928,10 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" dependencies = [ - "base16ct 1.0.0", + "base16ct", "serde", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - [[package]] name = "sha2" version = "0.11.0" @@ -1204,16 +960,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest 0.10.7", - "rand_core 0.6.4", -] - [[package]] name = "signature" version = "3.0.0" @@ -1230,16 +976,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45e14297decde697ddf377c25752aead0927d5cfc89c2684d2af96901a4ceeea" -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der 0.7.10", -] - [[package]] name = "spki" version = "0.8.0" @@ -1247,7 +983,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" dependencies = [ "base64ct", - "der 0.8.1", + "der", ] [[package]] @@ -1305,16 +1041,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "universal-hash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" -dependencies = [ - "crypto-common 0.1.7", - "subtle", -] - [[package]] name = "universal-hash" version = "0.6.1" @@ -1331,32 +1057,25 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - [[package]] name = "wnaf" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1" dependencies = [ - "ff 0.14.0", - "group 0.14.0", + "ff", + "group", "hybrid-array", ] [[package]] name = "x25519-dalek" -version = "2.0.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +checksum = "e7e8131a03190127fb2263afc72b322ecadae46b6ff8c6f399ff5d02f5559af6" dependencies = [ "curve25519-dalek", - "rand_core 0.6.4", - "serde", + "rand_core 0.10.1", "zeroize", ] @@ -1371,17 +1090,3 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index ce331f94..a60b0fb3 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -8,7 +8,7 @@ edition = "2024" cargo-fuzz = true [lints.rust] -unexpected_cfgs = { level = "warn", check-cfg = [ +unexpected_cfgs = { level = "deny", check-cfg = [ 'cfg(fuzzing)', 'cfg(rscrypto_internal_fuzzing)', ] } @@ -28,24 +28,24 @@ rscrypto = { path = "..", features = ["std", "checksums", "hashes", "auth", "aea # AEAD ascon-aead = { version = "0.6", default-features = false, features = ["alloc"] } aes-gcm = { version = "0.11", default-features = false, features = ["alloc", "aes"] } -aes-gcm-siv = { version = "0.11", default-features = false, features = ["alloc", "aes"] } +aes-gcm-siv = { version = "0.12", default-features = false, features = ["alloc", "aes"] } aegis = "0.9" chacha20poly1305 = { version = "0.11", default-features = false, features = ["alloc"] } # Auth -argon2 = { version = "0.6.0-rc.8", default-features = false, features = ["alloc"] } -ed25519-dalek = "2.2" +argon2 = { version = "0.5.3", default-features = false, features = ["alloc"] } +ed25519-dalek = "3.0" hmac = "0.13" hkdf = "0.13" p256 = { version = "0.14.0", default-features = false, features = ["ecdsa", "pkcs8", "std"] } -p384 = { version = "0.13.1", default-features = false, features = ["ecdsa", "pkcs8", "std"] } +p384 = { version = "0.14.0", default-features = false, features = ["ecdsa", "pkcs8", "std"] } pbkdf2 = "0.13" scrypt = { version = "0.12", default-features = false } -x25519-dalek = { version = "2.0.1", features = ["static_secrets"] } +x25519-dalek = { version = "3.0.0", features = ["static_secrets"] } # Hashes ascon-hash = "0.4" -blake2 = "0.11.0-rc.6" +blake2 = "0.10.6" blake3 = "1.8" cshake = { version = "0.2.1", default-features = false } digest = "0.11" diff --git a/fuzz/fuzz_targets/aead_aegis256.rs b/fuzz/fuzz_targets/aead_aegis256.rs index 44802516..09aaf1af 100644 --- a/fuzz/fuzz_targets/aead_aegis256.rs +++ b/fuzz/fuzz_targets/aead_aegis256.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_aes128gcm.rs b/fuzz/fuzz_targets/aead_aes128gcm.rs index 30d3a805..dcbcb90f 100644 --- a/fuzz/fuzz_targets/aead_aes128gcm.rs +++ b/fuzz/fuzz_targets/aead_aes128gcm.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_aes128gcmsiv.rs b/fuzz/fuzz_targets/aead_aes128gcmsiv.rs index 3276daf4..ea606ddd 100644 --- a/fuzz/fuzz_targets/aead_aes128gcmsiv.rs +++ b/fuzz/fuzz_targets/aead_aes128gcmsiv.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_aes256gcm.rs b/fuzz/fuzz_targets/aead_aes256gcm.rs index 691613d5..833a768c 100644 --- a/fuzz/fuzz_targets/aead_aes256gcm.rs +++ b/fuzz/fuzz_targets/aead_aes256gcm.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_aes256gcmsiv.rs b/fuzz/fuzz_targets/aead_aes256gcmsiv.rs index e182e649..f5aefce3 100644 --- a/fuzz/fuzz_targets/aead_aes256gcmsiv.rs +++ b/fuzz/fuzz_targets/aead_aes256gcmsiv.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_ascon128.rs b/fuzz/fuzz_targets/aead_ascon128.rs index e60350d4..7cb5625b 100644 --- a/fuzz/fuzz_targets/aead_ascon128.rs +++ b/fuzz/fuzz_targets/aead_ascon128.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_chacha20poly1305.rs b/fuzz/fuzz_targets/aead_chacha20poly1305.rs index 86673314..b18a79c9 100644 --- a/fuzz/fuzz_targets/aead_chacha20poly1305.rs +++ b/fuzz/fuzz_targets/aead_chacha20poly1305.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_nonce_counter.rs b/fuzz/fuzz_targets/aead_nonce_counter.rs index 35d353da..97ef6b9b 100644 --- a/fuzz/fuzz_targets/aead_nonce_counter.rs +++ b/fuzz/fuzz_targets/aead_nonce_counter.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_xchacha20poly1305.rs b/fuzz/fuzz_targets/aead_xchacha20poly1305.rs index 489e289e..2d0d6078 100644 --- a/fuzz/fuzz_targets/aead_xchacha20poly1305.rs +++ b/fuzz/fuzz_targets/aead_xchacha20poly1305.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_argon2d.rs b/fuzz/fuzz_targets/auth_argon2d.rs index a45efb2f..3d97698d 100644 --- a/fuzz/fuzz_targets/auth_argon2d.rs +++ b/fuzz/fuzz_targets/auth_argon2d.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_argon2i.rs b/fuzz/fuzz_targets/auth_argon2i.rs index 92a97153..06414a6f 100644 --- a/fuzz/fuzz_targets/auth_argon2i.rs +++ b/fuzz/fuzz_targets/auth_argon2i.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_argon2id.rs b/fuzz/fuzz_targets/auth_argon2id.rs index 7d86ce8d..82aaffa8 100644 --- a/fuzz/fuzz_targets/auth_argon2id.rs +++ b/fuzz/fuzz_targets/auth_argon2id.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_ed25519.rs b/fuzz/fuzz_targets/auth_ed25519.rs index 1447944c..bdc6c808 100644 --- a/fuzz/fuzz_targets/auth_ed25519.rs +++ b/fuzz/fuzz_targets/auth_ed25519.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_ed25519_verify.rs b/fuzz/fuzz_targets/auth_ed25519_verify.rs index 1feb01d4..a61f9316 100644 --- a/fuzz/fuzz_targets/auth_ed25519_verify.rs +++ b/fuzz/fuzz_targets/auth_ed25519_verify.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_hkdf_sha256.rs b/fuzz/fuzz_targets/auth_hkdf_sha256.rs index 7a356e3b..9beee577 100644 --- a/fuzz/fuzz_targets/auth_hkdf_sha256.rs +++ b/fuzz/fuzz_targets/auth_hkdf_sha256.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_hkdf_sha384.rs b/fuzz/fuzz_targets/auth_hkdf_sha384.rs index 41edb832..d48279d1 100644 --- a/fuzz/fuzz_targets/auth_hkdf_sha384.rs +++ b/fuzz/fuzz_targets/auth_hkdf_sha384.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_hmac_sha256.rs b/fuzz/fuzz_targets/auth_hmac_sha256.rs index 4249bbbf..a02999eb 100644 --- a/fuzz/fuzz_targets/auth_hmac_sha256.rs +++ b/fuzz/fuzz_targets/auth_hmac_sha256.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_hmac_sha384.rs b/fuzz/fuzz_targets/auth_hmac_sha384.rs index 3c867e00..e372fcb9 100644 --- a/fuzz/fuzz_targets/auth_hmac_sha384.rs +++ b/fuzz/fuzz_targets/auth_hmac_sha384.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_hmac_sha512.rs b/fuzz/fuzz_targets/auth_hmac_sha512.rs index 0205f50e..441a4a53 100644 --- a/fuzz/fuzz_targets/auth_hmac_sha512.rs +++ b/fuzz/fuzz_targets/auth_hmac_sha512.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_kmac256.rs b/fuzz/fuzz_targets/auth_kmac256.rs index c87e8e57..c33d9039 100644 --- a/fuzz/fuzz_targets/auth_kmac256.rs +++ b/fuzz/fuzz_targets/auth_kmac256.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_pbkdf2.rs b/fuzz/fuzz_targets/auth_pbkdf2.rs index ff7a4668..c70fea3d 100644 --- a/fuzz/fuzz_targets/auth_pbkdf2.rs +++ b/fuzz/fuzz_targets/auth_pbkdf2.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_phc.rs b/fuzz/fuzz_targets/auth_phc.rs index a77ff641..126149a9 100644 --- a/fuzz/fuzz_targets/auth_phc.rs +++ b/fuzz/fuzz_targets/auth_phc.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_scrypt.rs b/fuzz/fuzz_targets/auth_scrypt.rs index 177e0bf6..be949324 100644 --- a/fuzz/fuzz_targets/auth_scrypt.rs +++ b/fuzz/fuzz_targets/auth_scrypt.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_x25519.rs b/fuzz/fuzz_targets/auth_x25519.rs index 04ba47b1..36703851 100644 --- a/fuzz/fuzz_targets/auth_x25519.rs +++ b/fuzz/fuzz_targets/auth_x25519.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/checksum_crc.rs b/fuzz/fuzz_targets/checksum_crc.rs index 8ce3e7d4..092575b6 100644 --- a/fuzz/fuzz_targets/checksum_crc.rs +++ b/fuzz/fuzz_targets/checksum_crc.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/fast_rapidhash.rs b/fuzz/fuzz_targets/fast_rapidhash.rs index 6826985a..212c8422 100644 --- a/fuzz/fuzz_targets/fast_rapidhash.rs +++ b/fuzz/fuzz_targets/fast_rapidhash.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/fast_xxh3.rs b/fuzz/fuzz_targets/fast_xxh3.rs index e1967fa2..dbd78f4d 100644 --- a/fuzz/fuzz_targets/fast_xxh3.rs +++ b/fuzz/fuzz_targets/fast_xxh3.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_ascon.rs b/fuzz/fuzz_targets/hash_ascon.rs index 6a652898..bd00a12d 100644 --- a/fuzz/fuzz_targets/hash_ascon.rs +++ b/fuzz/fuzz_targets/hash_ascon.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_ascon_cxof.rs b/fuzz/fuzz_targets/hash_ascon_cxof.rs index bb51e4f9..3d93175d 100644 --- a/fuzz/fuzz_targets/hash_ascon_cxof.rs +++ b/fuzz/fuzz_targets/hash_ascon_cxof.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_blake2b.rs b/fuzz/fuzz_targets/hash_blake2b.rs index d9416c39..288c2abb 100644 --- a/fuzz/fuzz_targets/hash_blake2b.rs +++ b/fuzz/fuzz_targets/hash_blake2b.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_blake2s.rs b/fuzz/fuzz_targets/hash_blake2s.rs index 5cee46ce..e6b431df 100644 --- a/fuzz/fuzz_targets/hash_blake2s.rs +++ b/fuzz/fuzz_targets/hash_blake2s.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_blake3.rs b/fuzz/fuzz_targets/hash_blake3.rs index ad8d8210..b2835b66 100644 --- a/fuzz/fuzz_targets/hash_blake3.rs +++ b/fuzz/fuzz_targets/hash_blake3.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_blake3_derive.rs b/fuzz/fuzz_targets/hash_blake3_derive.rs index 0d92de67..609f2c3a 100644 --- a/fuzz/fuzz_targets/hash_blake3_derive.rs +++ b/fuzz/fuzz_targets/hash_blake3_derive.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_blake3_keyed.rs b/fuzz/fuzz_targets/hash_blake3_keyed.rs index adbde5eb..1daf7c28 100644 --- a/fuzz/fuzz_targets/hash_blake3_keyed.rs +++ b/fuzz/fuzz_targets/hash_blake3_keyed.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_cshake256.rs b/fuzz/fuzz_targets/hash_cshake256.rs index 5ebd4601..52c2bc46 100644 --- a/fuzz/fuzz_targets/hash_cshake256.rs +++ b/fuzz/fuzz_targets/hash_cshake256.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_sha2.rs b/fuzz/fuzz_targets/hash_sha2.rs index f7397733..306888bc 100644 --- a/fuzz/fuzz_targets/hash_sha2.rs +++ b/fuzz/fuzz_targets/hash_sha2.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_sha3.rs b/fuzz/fuzz_targets/hash_sha3.rs index 7d72a358..49b9f46d 100644 --- a/fuzz/fuzz_targets/hash_sha3.rs +++ b/fuzz/fuzz_targets/hash_sha3.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hex_parse.rs b/fuzz/fuzz_targets/hex_parse.rs index 45ee5985..8a947d3b 100644 --- a/fuzz/fuzz_targets/hex_parse.rs +++ b/fuzz/fuzz_targets/hex_parse.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/traits_io.rs b/fuzz/fuzz_targets/traits_io.rs index e741afd4..fe50fb54 100644 --- a/fuzz/fuzz_targets/traits_io.rs +++ b/fuzz/fuzz_targets/traits_io.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/support/src/lib.rs b/fuzz/support/src/lib.rs index ab1ff4f4..a2e0c9a5 100644 --- a/fuzz/support/src/lib.rs +++ b/fuzz/support/src/lib.rs @@ -51,12 +51,12 @@ where ); let mut files = fs::read_dir(corpus_dir) - .unwrap_or_else(|err| panic!("corpus replay: failed to read {}: {err}", corpus_dir.display())) - .map(|entry| entry.unwrap_or_else(|err| panic!("corpus replay: failed to read directory entry: {err}"))) + .expect("corpus replay directory must be readable") + .map(|entry| entry.expect("corpus replay directory entries must be readable")) .filter_map(|entry| { let file_type = entry .file_type() - .unwrap_or_else(|err| panic!("corpus replay: failed to inspect {}: {err}", entry.path().display())); + .expect("corpus replay directory entry metadata must be readable"); if file_type.is_file() || file_type.is_symlink() { Some(entry.path()) } else { @@ -74,7 +74,7 @@ where let mut count = 0usize; for path in files { - let data = fs::read(&path).unwrap_or_else(|err| panic!("corpus replay: failed to read {}: {err}", path.display())); + let data = fs::read(&path).expect("corpus replay input must be readable"); run(&data); count = count.checked_add(1).expect("corpus replay file count overflow"); } @@ -154,7 +154,7 @@ pub fn pad_salt_to(material: &[u8], filler: u8) -> [u8; N] { out[..N / 2].fill(filler); } else { for (i, slot) in out.iter_mut().enumerate() { - *slot = material[i % material.len()]; + *slot = material[i.rem_euclid(material.len())]; } } out @@ -258,7 +258,7 @@ pub fn assert_aead_forgery(cipher: &A, nonce: &A::Nonce, aad: &[u8], pl match target { 0 if !ct.is_empty() => { let mut forged = ct.clone(); - let idx = seed as usize % forged.len(); + let idx = usize::from(seed).rem_euclid(forged.len()); forged[idx] ^= 1u8 << (seed as u32 & 7); assert!( cipher.decrypt_in_place(nonce, aad, &mut forged, &tag).is_err(), @@ -267,7 +267,7 @@ pub fn assert_aead_forgery(cipher: &A, nonce: &A::Nonce, aad: &[u8], pl } 2 if !aad.is_empty() => { let mut forged_aad = aad.to_vec(); - let idx = seed as usize % forged_aad.len(); + let idx = usize::from(seed).rem_euclid(forged_aad.len()); forged_aad[idx] ^= 1u8 << (seed as u32 & 7); let mut ct_copy = ct.clone(); assert!( @@ -278,9 +278,9 @@ pub fn assert_aead_forgery(cipher: &A, nonce: &A::Nonce, aad: &[u8], pl _ => { let tag_ref = tag.as_ref(); let mut tag_bytes = tag_ref.to_vec(); - let idx = seed as usize % tag_bytes.len(); + let idx = usize::from(seed).rem_euclid(tag_bytes.len()); tag_bytes[idx] ^= 1u8 << (seed as u32 & 7); - let forged_tag = A::tag_from_slice(&tag_bytes).unwrap(); + let forged_tag = A::tag_from_slice(&tag_bytes).expect("forged tag preserves the algorithm tag length"); let mut ct_copy = ct.clone(); assert!( cipher.decrypt_in_place(nonce, aad, &mut ct_copy, &forged_tag).is_err(), diff --git a/fuzz/target_impls/aead_aegis256.rs b/fuzz/target_impls/aead_aegis256.rs index af12a9c0..4fc1f8c6 100644 --- a/fuzz/target_impls/aead_aegis256.rs +++ b/fuzz/target_impls/aead_aegis256.rs @@ -1,10 +1,10 @@ use rscrypto::{ - Aegis256, Aegis256Key, - aead::{Nonce256, expert::AeadWithNonce}, + Aegis256, Aegis256Key, + aead::{Nonce256, expert::AeadWithNonce}, }; use rscrypto_fuzz::{FuzzInput, assert_aead_forgery, assert_aead_roundtrip, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 32] = some_or_return!(input.bytes()); @@ -26,9 +26,14 @@ pub fn run(data: &[u8]) { let tag = cipher .encrypt_in_place(&nonce, aad, &mut ct) .expect("differential: rscrypto encrypt must succeed"); - let tag_arr: [u8; 16] = tag.as_ref().try_into().unwrap(); + let tag_arr: [u8; 16] = tag + .as_ref() + .try_into() + .expect("AEGIS-256 produces a 16-byte authentication tag"); let oracle = OracleAegis::<16>::new(&key_bytes, &nonce_bytes); - let pt = oracle.decrypt(&ct, &tag_arr, aad).unwrap(); + let pt = oracle + .decrypt(&ct, &tag_arr, aad) + .expect("oracle must accept the equivalent rscrypto ciphertext"); assert_eq!(pt, plaintext, "oracle failed to decrypt our ciphertext"); // oracle encrypt → rscrypto decrypt @@ -36,7 +41,12 @@ pub fn run(data: &[u8]) { let (oct, otag) = oracle_enc.encrypt(plaintext, aad); let mut buf = oct; cipher - .decrypt_in_place(&nonce, aad, &mut buf, &Aegis256::tag_from_slice(&otag).unwrap()) + .decrypt_in_place( + &nonce, + aad, + &mut buf, + &Aegis256::tag_from_slice(&otag).expect("oracle produces a 16-byte authentication tag"), + ) .expect("we failed to decrypt oracle ciphertext"); assert_eq!(buf, plaintext, "decrypt mismatch on oracle ciphertext"); } diff --git a/fuzz/target_impls/aead_aes128gcm.rs b/fuzz/target_impls/aead_aes128gcm.rs index b810880f..2a7f88b9 100644 --- a/fuzz/target_impls/aead_aes128gcm.rs +++ b/fuzz/target_impls/aead_aes128gcm.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 16] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 12] = some_or_return!(input.bytes()); @@ -18,14 +18,22 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ aes-gcm crate. use aes_gcm::aead::{Aead as _, KeyInit, Payload}; - let oracle = aes_gcm::Aes128Gcm::new_from_slice(&key_bytes).unwrap(); + let oracle = aes_gcm::Aes128Gcm::new_from_slice(&key_bytes).expect("AES-128-GCM accepts a 16-byte key"); let on = aes_gcm::Nonce::from(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(&on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(&on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/aead_aes128gcmsiv.rs b/fuzz/target_impls/aead_aes128gcmsiv.rs index 036ec8cc..3d0055bc 100644 --- a/fuzz/target_impls/aead_aes128gcmsiv.rs +++ b/fuzz/target_impls/aead_aes128gcmsiv.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 16] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 12] = some_or_return!(input.bytes()); @@ -18,14 +18,22 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ aes-gcm-siv crate. use aes_gcm_siv::aead::{Aead as _, KeyInit, Payload}; - let oracle = aes_gcm_siv::Aes128GcmSiv::new_from_slice(&key_bytes).unwrap(); - let on = aes_gcm_siv::Nonce::from_slice(&nonce_bytes); + let oracle = aes_gcm_siv::Aes128GcmSiv::new_from_slice(&key_bytes).expect("AES-128-GCM-SIV accepts a 16-byte key"); + let on = aes_gcm_siv::Nonce::from(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/aead_aes256gcm.rs b/fuzz/target_impls/aead_aes256gcm.rs index 73e0d142..40a49538 100644 --- a/fuzz/target_impls/aead_aes256gcm.rs +++ b/fuzz/target_impls/aead_aes256gcm.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 12] = some_or_return!(input.bytes()); @@ -18,14 +18,22 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ aes-gcm crate. use aes_gcm::aead::{Aead as _, KeyInit, Payload}; - let oracle = aes_gcm::Aes256Gcm::new_from_slice(&key_bytes).unwrap(); + let oracle = aes_gcm::Aes256Gcm::new_from_slice(&key_bytes).expect("AES-256-GCM accepts a 32-byte key"); let on = aes_gcm::Nonce::from(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(&on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(&on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/aead_aes256gcmsiv.rs b/fuzz/target_impls/aead_aes256gcmsiv.rs index 5c171acd..34ce2186 100644 --- a/fuzz/target_impls/aead_aes256gcmsiv.rs +++ b/fuzz/target_impls/aead_aes256gcmsiv.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 12] = some_or_return!(input.bytes()); @@ -18,14 +18,22 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ aes-gcm-siv crate. use aes_gcm_siv::aead::{Aead as _, KeyInit, Payload}; - let oracle = aes_gcm_siv::Aes256GcmSiv::new_from_slice(&key_bytes).unwrap(); - let on = aes_gcm_siv::Nonce::from_slice(&nonce_bytes); + let oracle = aes_gcm_siv::Aes256GcmSiv::new_from_slice(&key_bytes).expect("AES-256-GCM-SIV accepts a 32-byte key"); + let on = aes_gcm_siv::Nonce::from(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/aead_ascon128.rs b/fuzz/target_impls/aead_ascon128.rs index d2fd430e..fa6dc6df 100644 --- a/fuzz/target_impls/aead_ascon128.rs +++ b/fuzz/target_impls/aead_ascon128.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 16] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 16] = some_or_return!(input.bytes()); @@ -18,14 +18,22 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ ascon-aead crate. use ascon_aead::aead::{Aead as _, KeyInit, Payload, array::Array}; - let oracle = ascon_aead::AsconAead128::new_from_slice(&key_bytes).unwrap(); + let oracle = ascon_aead::AsconAead128::new_from_slice(&key_bytes).expect("Ascon-AEAD128 accepts a 16-byte key"); let on = Array(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(&on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(&on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/aead_chacha20poly1305.rs b/fuzz/target_impls/aead_chacha20poly1305.rs index 296d041c..039c0ab9 100644 --- a/fuzz/target_impls/aead_chacha20poly1305.rs +++ b/fuzz/target_impls/aead_chacha20poly1305.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 12] = some_or_return!(input.bytes()); @@ -18,14 +18,23 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ chacha20poly1305 crate. use chacha20poly1305::aead::{Aead as _, KeyInit, Payload}; - let oracle = chacha20poly1305::ChaCha20Poly1305::new_from_slice(&key_bytes).unwrap(); + let oracle = + chacha20poly1305::ChaCha20Poly1305::new_from_slice(&key_bytes).expect("ChaCha20-Poly1305 accepts a 32-byte key"); let on = chacha20poly1305::Nonce::from(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(&on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(&on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/aead_nonce_counter.rs b/fuzz/target_impls/aead_nonce_counter.rs index cd681fa2..b764fc48 100644 --- a/fuzz/target_impls/aead_nonce_counter.rs +++ b/fuzz/target_impls/aead_nonce_counter.rs @@ -1,6 +1,6 @@ use rscrypto::{ - Aes256Gcm, Aes256GcmKey, - aead::{NonceCounter, expert::AeadWithNonce}, + Aes256Gcm, Aes256GcmKey, + aead::{NonceCounter, expert::AeadWithNonce}, }; use rscrypto_fuzz::{FuzzInput, some_or_return, split_at_ratio}; @@ -87,7 +87,7 @@ fn assert_resume_equivalence(prefix: [u8; 4], advance: u32) { ); } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let prefix: [u8; 4] = some_or_return!(input.bytes()); @@ -101,12 +101,12 @@ pub fn run(data: &[u8]) { // are pinned by `aes_gcm_nonce_counter_exhausts_cleanly` and // `aes_gcm_nonce_counter_with_counter_rejects_max` in src/aead/nonce_counter.rs. let initial = u64::from_le_bytes(initial_bytes) - % NonceCounter::::MAX_MESSAGES.strict_sub(u64::from(MAX_NONCES_PER_ITER)); - let burst = (u32::from(burst_byte) % MAX_NONCES_PER_ITER).strict_add(1); + .rem_euclid(NonceCounter::::MAX_MESSAGES.strict_sub(u64::from(MAX_NONCES_PER_ITER))); + let burst = u32::from(burst_byte).rem_euclid(MAX_NONCES_PER_ITER).strict_add(1); // Resume-equivalence advance bounded so the fresh-counter loop runs // ≤ MAX_NONCES_PER_ITER times — exercised every iteration rather than // gated on a 10⁻¹² fuzzer probability. - let resume_advance = u32::from(burst_byte) % MAX_NONCES_PER_ITER; + let resume_advance = u32::from(burst_byte).rem_euclid(MAX_NONCES_PER_ITER); let cipher = Aes256Gcm::new(&Aes256GcmKey::from_bytes(key_bytes)); let (aad, plaintext) = split_at_ratio(rest, aad_split); diff --git a/fuzz/target_impls/aead_xchacha20poly1305.rs b/fuzz/target_impls/aead_xchacha20poly1305.rs index b0341567..5f08f3b9 100644 --- a/fuzz/target_impls/aead_xchacha20poly1305.rs +++ b/fuzz/target_impls/aead_xchacha20poly1305.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 24] = some_or_return!(input.bytes()); @@ -18,14 +18,23 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ chacha20poly1305 crate (XChaCha variant). use chacha20poly1305::aead::{Aead as _, KeyInit, Payload}; - let oracle = chacha20poly1305::XChaCha20Poly1305::new_from_slice(&key_bytes).unwrap(); + let oracle = + chacha20poly1305::XChaCha20Poly1305::new_from_slice(&key_bytes).expect("XChaCha20-Poly1305 accepts a 32-byte key"); let on = chacha20poly1305::XNonce::from(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(&on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(&on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/auth_argon2d.rs b/fuzz/target_impls/auth_argon2d.rs index 0b1c083d..2c10d2b9 100644 --- a/fuzz/target_impls/auth_argon2d.rs +++ b/fuzz/target_impls/auth_argon2d.rs @@ -1,7 +1,7 @@ use rscrypto::{Argon2Params, Argon2d}; use rscrypto_fuzz::{FuzzInput, pad_salt_to, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let pw_salt_split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); @@ -13,8 +13,7 @@ pub fn run(data: &[u8]) { let t = 1u32.strict_add(u32::from(t_byte) % 4); let out_len = 4u32.strict_add(u32::from(out_len_byte) % 29); - let params = Argon2Params::new(m_kib, t, 1) - .expect("params must be valid for fuzzer ranges"); + let params = Argon2Params::new(m_kib, t, 1).expect("params must be valid for fuzzer ranges"); // See `auth_argon2id.rs` for cost-parameter rationale. let (password, salt_material) = split_at_ratio(rest, pw_salt_split); diff --git a/fuzz/target_impls/auth_argon2i.rs b/fuzz/target_impls/auth_argon2i.rs index 799f2adf..0430dce9 100644 --- a/fuzz/target_impls/auth_argon2i.rs +++ b/fuzz/target_impls/auth_argon2i.rs @@ -1,7 +1,7 @@ use rscrypto::{Argon2Params, Argon2i}; use rscrypto_fuzz::{FuzzInput, pad_salt_to, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let pw_salt_split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); @@ -13,8 +13,7 @@ pub fn run(data: &[u8]) { let t = 1u32.strict_add(u32::from(t_byte) % 4); let out_len = 4u32.strict_add(u32::from(out_len_byte) % 29); - let params = Argon2Params::new(m_kib, t, 1) - .expect("params must be valid for fuzzer ranges"); + let params = Argon2Params::new(m_kib, t, 1).expect("params must be valid for fuzzer ranges"); // See `auth_argon2id.rs` for cost-parameter rationale. let (password, salt_material) = split_at_ratio(rest, pw_salt_split); diff --git a/fuzz/target_impls/auth_argon2id.rs b/fuzz/target_impls/auth_argon2id.rs index 57fef803..7e0bc4a8 100644 --- a/fuzz/target_impls/auth_argon2id.rs +++ b/fuzz/target_impls/auth_argon2id.rs @@ -1,7 +1,7 @@ use rscrypto::{Argon2Params, Argon2id}; use rscrypto_fuzz::{FuzzInput, pad_salt_to, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let pw_salt_split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); @@ -16,8 +16,7 @@ pub fn run(data: &[u8]) { let t = 1u32.strict_add(u32::from(t_byte) % 4); let out_len = 4u32.strict_add(u32::from(out_len_byte) % 29); // 4..=32 per RFC 9106 §3.1 - let params = Argon2Params::new(m_kib, t, 1) - .expect("params must be valid for fuzzer ranges"); + let params = Argon2Params::new(m_kib, t, 1).expect("params must be valid for fuzzer ranges"); let (password, salt_material) = split_at_ratio(rest, pw_salt_split); let salt_buf = pad_salt_to::<16>(salt_material, pw_salt_split); diff --git a/fuzz/target_impls/auth_ecdsa_sign.rs b/fuzz/target_impls/auth_ecdsa_sign.rs index fb1dd2e7..f796f4bc 100644 --- a/fuzz/target_impls/auth_ecdsa_sign.rs +++ b/fuzz/target_impls/auth_ecdsa_sign.rs @@ -7,7 +7,7 @@ use p384::ecdsa::{ use rscrypto::{EcdsaP256Keypair, EcdsaP256SecretKey, EcdsaP384Keypair, EcdsaP384SecretKey}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let selector = some_or_return!(input.byte()); @@ -25,15 +25,21 @@ fn run_p256(input: &mut FuzzInput<'_>) { let message = input.rest(); let public = keypair.public_key(); let signature = some_or_return!(keypair.try_sign(message).ok()); - let oracle_public = P256OracleVerifyingKey::from_sec1_bytes(&public.to_sec1_bytes()).expect("derived P-256 public key"); + let oracle_public = + P256OracleVerifyingKey::from_sec1_bytes(&public.to_sec1_bytes()).expect("derived P-256 public key"); let oracle_signature = P256OracleSignature::from_slice(signature.as_bytes()).expect("derived P-256 signature"); - assert!(public.verify(message, &signature).is_ok()); - assert!(P256Verifier::verify(&oracle_public, message, &oracle_signature).is_ok()); + public + .verify(message, &signature) + .expect("P-256 public key must verify its own signature"); + P256Verifier::verify(&oracle_public, message, &oracle_signature) + .expect("P-256 oracle must verify the equivalent signature"); let mut tampered = message.to_vec(); tampered.push(0x80); - assert!(public.verify(&tampered, &signature).is_err()); + let _verification_error = public + .verify(&tampered, &signature) + .expect_err("P-256 verification must reject a tampered message"); } fn run_p384(input: &mut FuzzInput<'_>) { @@ -43,13 +49,19 @@ fn run_p384(input: &mut FuzzInput<'_>) { let message = input.rest(); let public = keypair.public_key(); let signature = some_or_return!(keypair.try_sign(message).ok()); - let oracle_public = P384OracleVerifyingKey::from_sec1_bytes(&public.to_sec1_bytes()).expect("derived P-384 public key"); + let oracle_public = + P384OracleVerifyingKey::from_sec1_bytes(&public.to_sec1_bytes()).expect("derived P-384 public key"); let oracle_signature = P384OracleSignature::from_slice(signature.as_bytes()).expect("derived P-384 signature"); - assert!(public.verify(message, &signature).is_ok()); - assert!(P384Verifier::verify(&oracle_public, message, &oracle_signature).is_ok()); + public + .verify(message, &signature) + .expect("P-384 public key must verify its own signature"); + P384Verifier::verify(&oracle_public, message, &oracle_signature) + .expect("P-384 oracle must verify the equivalent signature"); let mut tampered = message.to_vec(); tampered.push(0x80); - assert!(public.verify(&tampered, &signature).is_err()); + let _verification_error = public + .verify(&tampered, &signature) + .expect_err("P-384 verification must reject a tampered message"); } diff --git a/fuzz/target_impls/auth_ecdsa_verify.rs b/fuzz/target_impls/auth_ecdsa_verify.rs index 2c04d88f..b38245c9 100644 --- a/fuzz/target_impls/auth_ecdsa_verify.rs +++ b/fuzz/target_impls/auth_ecdsa_verify.rs @@ -13,7 +13,7 @@ fn array_from_slice(bytes: &[u8]) -> [u8; N] { bytes.try_into().expect("oracle ECDSA signature length must match") } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let selector = some_or_return!(input.byte()); let derive_public = selector & 2 != 0; diff --git a/fuzz/target_impls/auth_ed25519.rs b/fuzz/target_impls/auth_ed25519.rs index fca0dbb8..fb1dae1c 100644 --- a/fuzz/target_impls/auth_ed25519.rs +++ b/fuzz/target_impls/auth_ed25519.rs @@ -1,7 +1,7 @@ use rscrypto::{Ed25519SecretKey, Ed25519Signature}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let message = input.rest(); diff --git a/fuzz/target_impls/auth_ed25519_verify.rs b/fuzz/target_impls/auth_ed25519_verify.rs index 5f5fc732..a1f63c87 100644 --- a/fuzz/target_impls/auth_ed25519_verify.rs +++ b/fuzz/target_impls/auth_ed25519_verify.rs @@ -1,7 +1,7 @@ use rscrypto::{Ed25519PublicKey, Ed25519Signature}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let public_bytes: [u8; 32] = some_or_return!(input.bytes()); let signature_bytes: [u8; 64] = some_or_return!(input.bytes()); diff --git a/fuzz/target_impls/auth_hkdf_sha256.rs b/fuzz/target_impls/auth_hkdf_sha256.rs index 8091ec39..bcd3b119 100644 --- a/fuzz/target_impls/auth_hkdf_sha256.rs +++ b/fuzz/target_impls/auth_hkdf_sha256.rs @@ -1,7 +1,7 @@ use rscrypto::HkdfSha256; use rscrypto_fuzz::{FuzzInput, assert_hkdf_against_oracle, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let salt_split: u8 = some_or_return!(input.byte()); let ikm_split: u8 = some_or_return!(input.byte()); @@ -14,7 +14,7 @@ pub fn run(data: &[u8]) { // Exercise both valid and over-length expand requests without allocating // unbounded buffers. - let out_len = usize::from(u16::from_le_bytes(out_len_bytes)) % HkdfSha256::MAX_OUTPUT_SIZE.strict_add(33); + let out_len = usize::from(u16::from_le_bytes(out_len_bytes)).rem_euclid(HkdfSha256::MAX_OUTPUT_SIZE.strict_add(33)); let hk = HkdfSha256::new(salt, ikm); let mut okm = vec![0u8; out_len]; diff --git a/fuzz/target_impls/auth_hkdf_sha384.rs b/fuzz/target_impls/auth_hkdf_sha384.rs index 8ba07a37..45c6be9d 100644 --- a/fuzz/target_impls/auth_hkdf_sha384.rs +++ b/fuzz/target_impls/auth_hkdf_sha384.rs @@ -1,7 +1,7 @@ use rscrypto::HkdfSha384; use rscrypto_fuzz::{FuzzInput, assert_hkdf_against_oracle, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let salt_split: u8 = some_or_return!(input.byte()); let ikm_split: u8 = some_or_return!(input.byte()); @@ -11,7 +11,7 @@ pub fn run(data: &[u8]) { let (salt, remainder) = split_at_ratio(rest, salt_split); let (ikm, info) = split_at_ratio(remainder, ikm_split); - let out_len = usize::from(u16::from_le_bytes(out_len_bytes)) % HkdfSha384::MAX_OUTPUT_SIZE.strict_add(33); + let out_len = usize::from(u16::from_le_bytes(out_len_bytes)).rem_euclid(HkdfSha384::MAX_OUTPUT_SIZE.strict_add(33)); let hk = HkdfSha384::new(salt, ikm); let mut okm = vec![0u8; out_len]; diff --git a/fuzz/target_impls/auth_hmac_sha256.rs b/fuzz/target_impls/auth_hmac_sha256.rs index 14983aa3..446ddc3a 100644 --- a/fuzz/target_impls/auth_hmac_sha256.rs +++ b/fuzz/target_impls/auth_hmac_sha256.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_mac_against_oracle, assert_mac_reset, assert_mac_streaming, some_or_return, split_at_ratio, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let key_split: u8 = some_or_return!(input.byte()); @@ -20,7 +20,7 @@ pub fn run(data: &[u8]) { assert_mac_against_oracle::(key, message, &tag, |key, msg| { use hmac::{Hmac, KeyInit, Mac as _}; - let mut oracle = as KeyInit>::new_from_slice(key).unwrap(); + let mut oracle = as KeyInit>::new_from_slice(key).expect("HMAC accepts keys of every length"); oracle.update(msg); oracle.finalize().into_bytes().to_vec() }); diff --git a/fuzz/target_impls/auth_hmac_sha384.rs b/fuzz/target_impls/auth_hmac_sha384.rs index 73aa94cf..3b221513 100644 --- a/fuzz/target_impls/auth_hmac_sha384.rs +++ b/fuzz/target_impls/auth_hmac_sha384.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_mac_against_oracle, assert_mac_reset, assert_mac_streaming, some_or_return, split_at_ratio, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let key_split: u8 = some_or_return!(input.byte()); @@ -19,7 +19,7 @@ pub fn run(data: &[u8]) { assert_mac_against_oracle::(key, message, &tag, |key, msg| { use hmac::{Hmac, KeyInit, Mac as _}; - let mut oracle = as KeyInit>::new_from_slice(key).unwrap(); + let mut oracle = as KeyInit>::new_from_slice(key).expect("HMAC accepts keys of every length"); oracle.update(msg); oracle.finalize().into_bytes().to_vec() }); diff --git a/fuzz/target_impls/auth_hmac_sha512.rs b/fuzz/target_impls/auth_hmac_sha512.rs index 9db49b07..5c2b44a4 100644 --- a/fuzz/target_impls/auth_hmac_sha512.rs +++ b/fuzz/target_impls/auth_hmac_sha512.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_mac_against_oracle, assert_mac_reset, assert_mac_streaming, some_or_return, split_at_ratio, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let key_split: u8 = some_or_return!(input.byte()); @@ -19,7 +19,7 @@ pub fn run(data: &[u8]) { assert_mac_against_oracle::(key, message, &tag, |key, msg| { use hmac::{Hmac, KeyInit, Mac as _}; - let mut oracle = as KeyInit>::new_from_slice(key).unwrap(); + let mut oracle = as KeyInit>::new_from_slice(key).expect("HMAC accepts keys of every length"); oracle.update(msg); oracle.finalize().into_bytes().to_vec() }); diff --git a/fuzz/target_impls/auth_kmac256.rs b/fuzz/target_impls/auth_kmac256.rs index 31854463..c4e7c75a 100644 --- a/fuzz/target_impls/auth_kmac256.rs +++ b/fuzz/target_impls/auth_kmac256.rs @@ -3,7 +3,11 @@ use rscrypto_fuzz::{FuzzInput, some_or_return, split_at_ratio}; fn encoded_string_len(len: usize) -> usize { let bits = len.strict_mul(8); - let width = ((usize::BITS - bits.leading_zeros()) as usize).div_ceil(8).max(1); + let width_bits = usize::BITS.strict_sub(bits.leading_zeros()); + let width = usize::try_from(width_bits) + .expect("the bit width fits usize") + .div_ceil(8) + .max(1); 1usize.strict_add(width).strict_add(len) } @@ -15,7 +19,7 @@ fn bytepad_is_aligned(rate: usize, segments: &[usize]) -> bool { encoded_len.is_multiple_of(rate) } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let key_split: u8 = some_or_return!(input.byte()); @@ -24,7 +28,7 @@ pub fn run(data: &[u8]) { let (key, remainder) = split_at_ratio(rest, key_split); let (custom, message) = split_at_ratio(remainder, split); - let out_len = (out_len_byte as usize % 128).strict_add(1); + let out_len = usize::from(out_len_byte).rem_euclid(128).strict_add(1); // Property: streaming equivalence let mut expected = vec![0u8; out_len]; diff --git a/fuzz/target_impls/auth_mlkem1024.rs b/fuzz/target_impls/auth_mlkem1024.rs index 698c8138..fa6de78f 100644 --- a/fuzz/target_impls/auth_mlkem1024.rs +++ b/fuzz/target_impls/auth_mlkem1024.rs @@ -3,7 +3,7 @@ use rscrypto::{ }; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_random: [u8; MlKem1024::KEY_GENERATION_RANDOM_SIZE] = some_or_return!(input.bytes()); let encaps_random: [u8; MlKem1024::ENCAPSULATION_RANDOM_SIZE] = some_or_return!(input.bytes()); @@ -27,9 +27,9 @@ pub fn run(data: &[u8]) { ); let parse_material = input.rest(); - let _ = MlKem1024EncapsulationKey::try_from_slice(parse_material); - let _ = MlKem1024DecapsulationKey::try_from_slice(parse_material); - let _ = MlKem1024Ciphertext::try_from_slice(parse_material); + let _encapsulation_key_result = MlKem1024EncapsulationKey::try_from_slice(parse_material); + let _decapsulation_key_result = MlKem1024DecapsulationKey::try_from_slice(parse_material); + let _ciphertext_result = MlKem1024Ciphertext::try_from_slice(parse_material); let Some(byte_idx) = input.byte() else { return; @@ -39,7 +39,7 @@ pub fn run(data: &[u8]) { }; let mut modified = ciphertext.to_bytes(); - modified[byte_idx as usize % MlKem1024::CIPHERTEXT_SIZE] ^= 1u8 << (bit_idx & 7); + modified[usize::from(byte_idx).rem_euclid(MlKem1024::CIPHERTEXT_SIZE)] ^= 1u8.strict_shl(u32::from(bit_idx & 7)); let rejected = MlKem1024::decapsulate(&dk, &MlKem1024Ciphertext::from_bytes(modified)) .expect("ML-KEM implicit rejection returns a shared secret"); assert!( diff --git a/fuzz/target_impls/auth_mlkem512.rs b/fuzz/target_impls/auth_mlkem512.rs index 3722449a..23d7b56a 100644 --- a/fuzz/target_impls/auth_mlkem512.rs +++ b/fuzz/target_impls/auth_mlkem512.rs @@ -3,7 +3,7 @@ use rscrypto::{ }; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_random: [u8; MlKem512::KEY_GENERATION_RANDOM_SIZE] = some_or_return!(input.bytes()); let encaps_random: [u8; MlKem512::ENCAPSULATION_RANDOM_SIZE] = some_or_return!(input.bytes()); @@ -27,9 +27,9 @@ pub fn run(data: &[u8]) { ); let parse_material = input.rest(); - let _ = MlKem512EncapsulationKey::try_from_slice(parse_material); - let _ = MlKem512DecapsulationKey::try_from_slice(parse_material); - let _ = MlKem512Ciphertext::try_from_slice(parse_material); + let _encapsulation_key_result = MlKem512EncapsulationKey::try_from_slice(parse_material); + let _decapsulation_key_result = MlKem512DecapsulationKey::try_from_slice(parse_material); + let _ciphertext_result = MlKem512Ciphertext::try_from_slice(parse_material); let Some(byte_idx) = input.byte() else { return; @@ -39,7 +39,7 @@ pub fn run(data: &[u8]) { }; let mut modified = ciphertext.to_bytes(); - modified[byte_idx as usize % MlKem512::CIPHERTEXT_SIZE] ^= 1u8 << (bit_idx & 7); + modified[usize::from(byte_idx).rem_euclid(MlKem512::CIPHERTEXT_SIZE)] ^= 1u8.strict_shl(u32::from(bit_idx & 7)); let rejected = MlKem512::decapsulate(&dk, &MlKem512Ciphertext::from_bytes(modified)) .expect("ML-KEM implicit rejection returns a shared secret"); assert!( diff --git a/fuzz/target_impls/auth_mlkem768.rs b/fuzz/target_impls/auth_mlkem768.rs index e8810455..f5aabf80 100644 --- a/fuzz/target_impls/auth_mlkem768.rs +++ b/fuzz/target_impls/auth_mlkem768.rs @@ -3,7 +3,7 @@ use rscrypto::{ }; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_random: [u8; MlKem768::KEY_GENERATION_RANDOM_SIZE] = some_or_return!(input.bytes()); let encaps_random: [u8; MlKem768::ENCAPSULATION_RANDOM_SIZE] = some_or_return!(input.bytes()); @@ -27,9 +27,9 @@ pub fn run(data: &[u8]) { ); let parse_material = input.rest(); - let _ = MlKem768EncapsulationKey::try_from_slice(parse_material); - let _ = MlKem768DecapsulationKey::try_from_slice(parse_material); - let _ = MlKem768Ciphertext::try_from_slice(parse_material); + let _encapsulation_key_result = MlKem768EncapsulationKey::try_from_slice(parse_material); + let _decapsulation_key_result = MlKem768DecapsulationKey::try_from_slice(parse_material); + let _ciphertext_result = MlKem768Ciphertext::try_from_slice(parse_material); let Some(byte_idx) = input.byte() else { return; @@ -39,7 +39,7 @@ pub fn run(data: &[u8]) { }; let mut modified = ciphertext.to_bytes(); - modified[byte_idx as usize % MlKem768::CIPHERTEXT_SIZE] ^= 1u8 << (bit_idx & 7); + modified[usize::from(byte_idx).rem_euclid(MlKem768::CIPHERTEXT_SIZE)] ^= 1u8.strict_shl(u32::from(bit_idx & 7)); let rejected = MlKem768::decapsulate(&dk, &MlKem768Ciphertext::from_bytes(modified)) .expect("ML-KEM implicit rejection returns a shared secret"); assert!( diff --git a/fuzz/target_impls/auth_pbkdf2.rs b/fuzz/target_impls/auth_pbkdf2.rs index dd6c921f..a02ed27c 100644 --- a/fuzz/target_impls/auth_pbkdf2.rs +++ b/fuzz/target_impls/auth_pbkdf2.rs @@ -1,7 +1,7 @@ use rscrypto::{Pbkdf2Sha256, Pbkdf2Sha512}; use rscrypto_fuzz::{FuzzInput, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); @@ -9,20 +9,25 @@ pub fn run(data: &[u8]) { let data = input.rest(); let (password, salt) = split_at_ratio(data, split); - let out_len = (out_len_byte as usize % 96).strict_add(1); - let iterations = (u32::from(u16::from_le_bytes(iterations_bytes)) % 64).strict_add(1); + let out_len = usize::from(out_len_byte).rem_euclid(96).strict_add(1); + let iterations = u32::from(u16::from_le_bytes(iterations_bytes)) + .rem_euclid(64) + .strict_add(1); let mut ours_256 = vec![0u8; out_len]; let mut ours_256_state = vec![0u8; out_len]; - Pbkdf2Sha256::derive_key_primitive(password, salt, iterations, &mut ours_256).unwrap(); + Pbkdf2Sha256::derive_key_primitive(password, salt, iterations, &mut ours_256) + .expect("the bounded PBKDF2-SHA-256 request is valid"); Pbkdf2Sha256::new(password) .derive(salt, iterations, &mut ours_256_state) - .unwrap(); + .expect("the bounded PBKDF2-SHA-256 request is valid"); assert_eq!(ours_256, ours_256_state, "pbkdf2-sha256 state reuse mismatch"); - assert!(Pbkdf2Sha256::verify_password_primitive(password, salt, iterations, &ours_256).is_ok()); + Pbkdf2Sha256::verify_password_primitive(password, salt, iterations, &ours_256) + .expect("PBKDF2-SHA-256 must verify its own output"); let mut wrong_256 = ours_256.clone(); wrong_256[0] ^= 1; - assert!(Pbkdf2Sha256::verify_password_primitive(password, salt, iterations, &wrong_256).is_err()); + let _verification_error = Pbkdf2Sha256::verify_password_primitive(password, salt, iterations, &wrong_256) + .expect_err("PBKDF2-SHA-256 must reject a corrupted output"); let mut oracle_256 = vec![0u8; out_len]; pbkdf2::pbkdf2_hmac::(password, salt, iterations, &mut oracle_256); @@ -30,15 +35,18 @@ pub fn run(data: &[u8]) { let mut ours_512 = vec![0u8; out_len]; let mut ours_512_state = vec![0u8; out_len]; - Pbkdf2Sha512::derive_key_primitive(password, salt, iterations, &mut ours_512).unwrap(); + Pbkdf2Sha512::derive_key_primitive(password, salt, iterations, &mut ours_512) + .expect("the bounded PBKDF2-SHA-512 request is valid"); Pbkdf2Sha512::new(password) .derive(salt, iterations, &mut ours_512_state) - .unwrap(); + .expect("the bounded PBKDF2-SHA-512 request is valid"); assert_eq!(ours_512, ours_512_state, "pbkdf2-sha512 state reuse mismatch"); - assert!(Pbkdf2Sha512::verify_password_primitive(password, salt, iterations, &ours_512).is_ok()); + Pbkdf2Sha512::verify_password_primitive(password, salt, iterations, &ours_512) + .expect("PBKDF2-SHA-512 must verify its own output"); let mut wrong_512 = ours_512.clone(); wrong_512[0] ^= 1; - assert!(Pbkdf2Sha512::verify_password_primitive(password, salt, iterations, &wrong_512).is_err()); + let _verification_error = Pbkdf2Sha512::verify_password_primitive(password, salt, iterations, &wrong_512) + .expect_err("PBKDF2-SHA-512 must reject a corrupted output"); let mut oracle_512 = vec![0u8; out_len]; pbkdf2::pbkdf2_hmac::(password, salt, iterations, &mut oracle_512); diff --git a/fuzz/target_impls/auth_phc.rs b/fuzz/target_impls/auth_phc.rs index 0bced237..c3da30cf 100644 --- a/fuzz/target_impls/auth_phc.rs +++ b/fuzz/target_impls/auth_phc.rs @@ -4,24 +4,18 @@ // canonical record remains cheap while malformed or over-budget inputs // exercise the parser and approval boundary at full throughput. -use rscrypto::{ - Argon2Params, Argon2idPassword, ScryptParams, ScryptPassword, -}; +use rscrypto::{Argon2Params, Argon2idPassword, ScryptParams, ScryptPassword}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let split = data.len() / 2; let (password, encoded_bytes) = data.split_at(split); let encoded = String::from_utf8_lossy(encoded_bytes); - let argon2 = Argon2idPassword::new( - Argon2Params::new(8, 1, 1).expect("fixed Argon2 fuzz profile is valid"), - ) - .expect("fixed Argon2 fuzz profile fits the target"); - let scrypt = ScryptPassword::new( - ScryptParams::new(1, 1, 1).expect("fixed scrypt fuzz profile is valid"), - ) - .expect("fixed scrypt fuzz profile fits the target"); + let argon2 = Argon2idPassword::new(Argon2Params::new(8, 1, 1).expect("fixed Argon2 fuzz profile is valid")) + .expect("fixed Argon2 fuzz profile fits the target"); + let scrypt = ScryptPassword::new(ScryptParams::new(1, 1, 1).expect("fixed scrypt fuzz profile is valid")) + .expect("fixed scrypt fuzz profile fits the target"); - let _ = argon2.verify_password(password, &encoded); - let _ = scrypt.verify_password(password, &encoded); + let _argon2_result = argon2.verify_password(password, &encoded); + let _scrypt_result = scrypt.verify_password(password, &encoded); } diff --git a/fuzz/target_impls/auth_rsa_import.rs b/fuzz/target_impls/auth_rsa_import.rs index b63c8ca6..6e71a2a5 100644 --- a/fuzz/target_impls/auth_rsa_import.rs +++ b/fuzz/target_impls/auth_rsa_import.rs @@ -1,10 +1,10 @@ #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] -use rscrypto_fuzz::{FuzzInput, some_or_return}; -#[cfg(any(fuzzing, rscrypto_internal_fuzzing))] use rscrypto::auth::rsa::fuzz_rsa_import_der; +#[cfg(any(fuzzing, rscrypto_internal_fuzzing))] +use rscrypto_fuzz::{FuzzInput, some_or_return}; #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let format = some_or_return!(input.byte()); let (der, expected) = if input.rest().first().copied() == Some(b'V') { @@ -15,12 +15,15 @@ pub fn run(data: &[u8]) { let accepted = fuzz_rsa_import_der(format, &der); if let Some(expected) = expected { - assert_eq!(accepted, expected, "generated RSA private-key import fixture expectation drifted"); + assert_eq!( + accepted, expected, + "generated RSA private-key import fixture expectation drifted" + ); } } #[cfg(not(any(fuzzing, rscrypto_internal_fuzzing)))] -pub fn run(_data: &[u8]) {} +pub(super) fn run(_data: &[u8]) {} #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] fn decoded_der(input: &[u8]) -> Vec { @@ -57,9 +60,21 @@ fn hex_value(byte: u8) -> Option { fn valid_private_key_der(format: u8, control: &[u8]) -> (Vec, Option) { let control = control.strip_suffix(b"\n").unwrap_or(control); let pkcs1 = match control.first().copied() { - Some(b'P') => pkcs1_private_key_der_with_crt(&[1], &hex_to_vec(RSA_PRIVATE_EXPONENT_Q_HEX), &hex_to_vec(RSA_PRIVATE_COEFFICIENT_HEX)), - Some(b'Q') => pkcs1_private_key_der_with_crt(&hex_to_vec(RSA_PRIVATE_EXPONENT_P_HEX), &[1], &hex_to_vec(RSA_PRIVATE_COEFFICIENT_HEX)), - Some(b'C') => pkcs1_private_key_der_with_crt(&hex_to_vec(RSA_PRIVATE_EXPONENT_P_HEX), &hex_to_vec(RSA_PRIVATE_EXPONENT_Q_HEX), &[1]), + Some(b'P') => pkcs1_private_key_der_with_crt( + &[1], + &hex_to_vec(RSA_PRIVATE_EXPONENT_Q_HEX), + &hex_to_vec(RSA_PRIVATE_COEFFICIENT_HEX), + ), + Some(b'Q') => pkcs1_private_key_der_with_crt( + &hex_to_vec(RSA_PRIVATE_EXPONENT_P_HEX), + &[1], + &hex_to_vec(RSA_PRIVATE_COEFFICIENT_HEX), + ), + Some(b'C') => pkcs1_private_key_der_with_crt( + &hex_to_vec(RSA_PRIVATE_EXPONENT_P_HEX), + &hex_to_vec(RSA_PRIVATE_EXPONENT_Q_HEX), + &[1], + ), Some(b'N') => pkcs1_private_key_der_with_noncanonical_version(), _ => valid_pkcs1_private_key_der(), }; @@ -94,7 +109,7 @@ fn valid_private_key_der(format: u8, control: &[u8]) -> (Vec, Option) } #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] -pub fn valid_pkcs1_private_key_der() -> Vec { +pub(super) fn valid_pkcs1_private_key_der() -> Vec { pkcs1_private_key_der_with_crt( &hex_to_vec(RSA_PRIVATE_EXPONENT_P_HEX), &hex_to_vec(RSA_PRIVATE_EXPONENT_Q_HEX), @@ -143,7 +158,11 @@ fn private_key_der_for_format(format: u8, pkcs1: &[u8]) -> Vec { #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] fn valid_pkcs8_private_key_der(pkcs1: &[u8]) -> Vec { - sequence(&[integer_unsigned(&[0]), algorithm_identifier(RSA_ENCRYPTION_OID, Some(&der_null())), tlv(0x04, pkcs1)]) + sequence(&[ + integer_unsigned(&[0]), + algorithm_identifier(RSA_ENCRYPTION_OID, Some(&der_null())), + tlv(0x04, pkcs1), + ]) } #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] diff --git a/fuzz/target_impls/auth_rsa_private_ops.rs b/fuzz/target_impls/auth_rsa_private_ops.rs index 706422a4..6b0168c0 100644 --- a/fuzz/target_impls/auth_rsa_private_ops.rs +++ b/fuzz/target_impls/auth_rsa_private_ops.rs @@ -1,13 +1,12 @@ #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] use rscrypto::{ - RsaEncryptionError, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKeyPolicy, - RsaSignatureProfile, + RsaBlindingPair, RsaEncryptionError, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, + RsaPublicKeyPolicy, RsaSignatureProfile, }; #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] use rscrypto_fuzz::{FuzzInput, some_or_return, split_at_ratio}; #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] -#[allow(dead_code)] #[path = "auth_rsa_import.rs"] mod rsa_import_fixture; @@ -23,7 +22,7 @@ const PSS_PROFILES: [RsaPssProfile; 3] = [RsaPssProfile::Sha256, RsaPssProfile:: const OAEP_PROFILES: [RsaOaepProfile; 3] = [RsaOaepProfile::Sha256, RsaOaepProfile::Sha384, RsaOaepProfile::Sha512]; #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let mode = some_or_return!(input.byte()); let selector = some_or_return!(input.byte()); @@ -37,12 +36,12 @@ pub fn run(data: &[u8]) { .expect("fuzz RSA private-key fixture must parse"); let (one, one_inverse) = factor_one(key.signature_len()); - match mode % 10 { + match mode.rem_euclid(10) { 0 => { let profile = PKCS1_PROFILES[usize::from(selector) % PKCS1_PROFILES.len()]; let mut signature = vec![0u8; key.signature_len()]; key - .sign_pkcs1v15_with_blinding_factor(profile, left, &one, &one_inverse, &mut signature) + .sign_pkcs1v15_with_blinding_factor(profile, left, RsaBlindingPair::new(&one, &one_inverse), &mut signature) .expect("fixture RSA-PKCS1-v1_5 signing must succeed"); key .public_key() @@ -54,7 +53,13 @@ pub fn run(data: &[u8]) { let salt = bounded_slice(right, profile.digest_len()); let mut signature = vec![0u8; key.signature_len()]; key - .sign_pss_with_salt_and_blinding_factor(profile, left, salt, &one, &one_inverse, &mut signature) + .sign_pss_with_salt_and_blinding_factor( + profile, + left, + salt, + RsaBlindingPair::new(&one, &one_inverse), + &mut signature, + ) .expect("fixture RSA-PSS signing must succeed"); key .public_key() @@ -73,7 +78,13 @@ pub fn run(data: &[u8]) { .expect("fixture RSA-OAEP encryption must succeed for bounded message"); let mut plaintext = vec![0u8; key.signature_len()]; let plaintext_len = key - .decrypt_oaep_with_blinding_factor(profile, label, &ciphertext, &one, &one_inverse, &mut plaintext) + .decrypt_oaep_with_blinding_factor( + profile, + label, + &ciphertext, + RsaBlindingPair::new(&one, &one_inverse), + &mut plaintext, + ) .expect("self-produced RSA-OAEP ciphertext must decrypt"); assert_eq!(&plaintext[..plaintext_len], message); } @@ -84,8 +95,7 @@ pub fn run(data: &[u8]) { .sign_pkcs1v15_with_blinding_factor( profile.pkcs1v15_profile().expect("profile is PKCS1-v1_5"), left, - &one, - &one_inverse, + RsaBlindingPair::new(&one, &one_inverse), &mut signature, ) .expect("typed fixture RSA-PKCS1-v1_5 signing must succeed"); @@ -98,30 +108,47 @@ pub fn run(data: &[u8]) { let profile = OAEP_PROFILES[usize::from(selector) % OAEP_PROFILES.len()]; let mut plaintext = vec![0u8; key.signature_len()]; let ciphertext = full_width_candidate(left, key.signature_len()); - let _ = key.decrypt_oaep_with_blinding_factor(profile, right, &ciphertext, &one, &one_inverse, &mut plaintext); + let _decryption_result = key.decrypt_oaep_with_blinding_factor( + profile, + right, + &ciphertext, + RsaBlindingPair::new(&one, &one_inverse), + &mut plaintext, + ); } 5 => { let profile = PKCS1_PROFILES[usize::from(selector) % PKCS1_PROFILES.len()]; let mut signature = vec![0u8; key.signature_len()]; let bad_factor = full_width_candidate(left, key.signature_len()); - let _ = key.sign_pkcs1v15_with_blinding_factor(profile, right, &bad_factor, &one_inverse, &mut signature); + let _signing_result = key.sign_pkcs1v15_with_blinding_factor( + profile, + right, + RsaBlindingPair::new(&bad_factor, &one_inverse), + &mut signature, + ); } 6 => { let profile = PSS_PROFILES[usize::from(selector) % PSS_PROFILES.len()]; let mut short_signature = vec![0u8; key.signature_len().saturating_sub(1)]; - assert!(key - .sign_pss_with_salt_and_blinding_factor(profile, left, right, &one, &one_inverse, &mut short_signature) - .is_err()); + let _signing_error = key + .sign_pss_with_salt_and_blinding_factor( + profile, + left, + right, + RsaBlindingPair::new(&one, &one_inverse), + &mut short_signature, + ) + .expect_err("RSA-PSS signing must reject a short output buffer"); } 7 => { let profile = OAEP_PROFILES[usize::from(selector) % OAEP_PROFILES.len()]; let mut ciphertext = vec![0u8; key.signature_len()]; - assert!(key + let _encryption_error = key .public_key() .encrypt_oaep_with_random_fill(profile, left, right, &mut ciphertext, |_| { Err(RsaEncryptionError::EntropyUnavailable) }) - .is_err()); + .expect_err("RSA-OAEP encryption must propagate entropy failure"); } 8 => { let message = bounded_slice(right, pkcs1v15_message_limit(&key)); @@ -133,21 +160,25 @@ pub fn run(data: &[u8]) { .expect("fixture RSAES-PKCS1-v1_5 encryption must succeed for bounded message"); let mut plaintext = vec![0u8; key.signature_len()]; let plaintext_len = key - .decrypt_pkcs1v15_with_blinding_factor(&ciphertext, &one, &one_inverse, &mut plaintext) + .decrypt_pkcs1v15_with_blinding_factor(&ciphertext, RsaBlindingPair::new(&one, &one_inverse), &mut plaintext) .expect("self-produced RSAES-PKCS1-v1_5 ciphertext must decrypt"); assert_eq!(&plaintext[..plaintext_len], message); } 9 => { let mut plaintext = vec![0u8; key.signature_len()]; let ciphertext = full_width_candidate(left, key.signature_len()); - let _ = key.decrypt_pkcs1v15_with_blinding_factor(&ciphertext, &one, &one_inverse, &mut plaintext); + let _decryption_result = key.decrypt_pkcs1v15_with_blinding_factor( + &ciphertext, + RsaBlindingPair::new(&one, &one_inverse), + &mut plaintext, + ); } - _ => unreachable!("mode modulo 10 is always in 0..10"), + _ => {} } } #[cfg(not(any(fuzzing, rscrypto_internal_fuzzing)))] -pub fn run(_data: &[u8]) {} +pub(super) fn run(_data: &[u8]) {} #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] fn factor_one(len: usize) -> (Vec, Vec) { @@ -191,7 +222,8 @@ fn fill_random_from(bytes: &[u8]) -> impl FnMut(&mut [u8]) -> Result<(), RsaEncr #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] fn oaep_message_limit(key: &RsaPrivateKey, profile: RsaOaepProfile) -> usize { - key.signature_len() + key + .signature_len() .saturating_sub(profile.digest_len().saturating_mul(2)) .saturating_sub(2) } @@ -213,10 +245,7 @@ fn pkcs1v15_message_limit(key: &RsaPrivateKey) -> usize { #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] fn pkcs1v15_seed(key: &RsaPrivateKey, message_len: usize, selector: u8, left: &[u8], right: &[u8]) -> Vec { - let len = key - .signature_len() - .saturating_sub(message_len) - .saturating_sub(3); + let len = key.signature_len().saturating_sub(message_len).saturating_sub(3); let mut seed = vec![selector.wrapping_add(1).max(1); len]; for (index, byte) in left.iter().chain(right.iter()).copied().enumerate() { let seed_len = seed.len(); diff --git a/fuzz/target_impls/auth_rsa_protocol.rs b/fuzz/target_impls/auth_rsa_protocol.rs index 4c51e398..152b95ff 100644 --- a/fuzz/target_impls/auth_rsa_protocol.rs +++ b/fuzz/target_impls/auth_rsa_protocol.rs @@ -28,8 +28,8 @@ const X509_PSS_DEFAULT_SHA1_ALGORITHM: &[u8] = &[ const X509_MALFORMED_INDEFINITE_SEQUENCE: &[u8] = &[0x30, 0x80, 0x00, 0x00]; const JWT_ALGS: [&str; 16] = [ - "PS256", "PS384", "PS512", "RS256", "RS384", "RS512", "none", "HS256", "ES256", "EdDSA", "ps256", "", "PS1", - "RS1", "PS256\0", "RS256 ", + "PS256", "PS384", "PS512", "RS256", "RS384", "RS512", "none", "HS256", "ES256", "EdDSA", "ps256", "", "PS1", "RS1", + "PS256\0", "RS256 ", ]; const COSE_ALGORITHMS: [i64; 16] = [ -37, @@ -50,8 +50,8 @@ const COSE_ALGORITHMS: [i64; 16] = [ i64::MAX - 1, ]; const TLS_SCHEMES: [u16; 19] = [ - 0x0804, 0x0805, 0x0806, 0x0809, 0x080a, 0x080b, 0x0401, 0x0501, 0x0601, 0x0101, 0x0201, 0x0420, - 0x0520, 0x0620, 0x0301, 0x0203, 0x0403, 0, 0xffff, + 0x0804, 0x0805, 0x0806, 0x0809, 0x080a, 0x080b, 0x0401, 0x0501, 0x0601, 0x0101, 0x0201, 0x0420, 0x0520, 0x0620, + 0x0301, 0x0203, 0x0403, 0, 0xffff, ]; const X509_ALGORITHMS: [&[u8]; 6] = [ X509_PSS_SHA256_ALGORITHM, @@ -69,7 +69,7 @@ fn signature_candidate(material: &[u8], len: usize) -> Vec { } for (index, byte) in out.iter_mut().enumerate() { - *byte = material[index % material.len()]; + *byte = material[index.rem_euclid(material.len())]; } out } @@ -80,10 +80,12 @@ fn selected_signature<'a>(material: &'a [u8], full_width: &'a [u8], selector: u8 #[inline] fn select(items: &[T], selector: u8) -> &T { - &items[(selector as usize) % items.len()] + items + .get(usize::from(selector).rem_euclid(items.len())) + .expect("RSA protocol selector tables are nonempty") } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let mode = some_or_return!(input.byte()); let selector = some_or_return!(input.byte()); @@ -96,7 +98,7 @@ pub fn run(data: &[u8]) { let signature = selected_signature(signature_material, &full_width_signature, selector); let mut scratch = key.public_scratch(); - match mode % 14 { + match mode.rem_euclid(14) { 0 => { key .jwt_verifier(RsaJwtAlgorithm::Ps256) @@ -134,12 +136,15 @@ pub fn run(data: &[u8]) { .expect("valid TLS certificate rsa_pkcs1_sha256 fixture must verify"); } 5 => { - let _ = key - .jwt_verifier(RsaJwtAlgorithm::Ps256) - .verify_with_scratch(select(&JWT_ALGS, selector), message, signature, &mut scratch); + let _verification_result = key.jwt_verifier(RsaJwtAlgorithm::Ps256).verify_with_scratch( + select(&JWT_ALGS, selector), + message, + signature, + &mut scratch, + ); } 6 => { - let _ = key.verify_cose_algorithm_id_with_scratch( + let _verification_result = key.verify_cose_algorithm_id_with_scratch( *select(&COSE_ALGORITHMS, selector), message, signature, @@ -147,7 +152,7 @@ pub fn run(data: &[u8]) { ); } 7 => { - let _ = x509_key.verify_signature_from_x509_algorithm_der_with_scratch( + let _verification_result = x509_key.verify_signature_from_x509_algorithm_der_with_scratch( select(&X509_ALGORITHMS, selector), message, signature, @@ -155,7 +160,7 @@ pub fn run(data: &[u8]) { ); } 8 => { - let _ = + let _verification_result = x509_key.verify_signature_from_x509_algorithm_der_with_scratch(message, MESSAGE_PSS, signature, &mut scratch); } 9 => { @@ -164,7 +169,8 @@ pub fn run(data: &[u8]) { } else { u16::from_be_bytes([selector, split]) }; - let _ = x509_key.verify_tls13_signature_scheme_with_scratch(scheme, message, signature, &mut scratch); + let _verification_result = + x509_key.verify_tls13_signature_scheme_with_scratch(scheme, message, signature, &mut scratch); } 10 => { let scheme = if selector & 1 == 0 { @@ -172,18 +178,20 @@ pub fn run(data: &[u8]) { } else { u16::from_be_bytes([selector, split]) }; - let _ = x509_key.verify_tls_certificate_signature_scheme_with_scratch(scheme, message, signature, &mut scratch); + let _verification_result = + x509_key.verify_tls_certificate_signature_scheme_with_scratch(scheme, message, signature, &mut scratch); } 11 => { - let _ = RsaSignatureProfile::from_x509_signature_algorithm_der(message); + let _profile_result = RsaSignatureProfile::from_x509_signature_algorithm_der(message); } 12 => { - let _ = RsaSignatureProfile::from_cose_algorithm_id(*select(&COSE_ALGORITHMS, selector)); - let _ = RsaSignatureProfile::from_tls13_signature_scheme(u16::from_be_bytes([selector, split])); - let _ = RsaSignatureProfile::from_tls_certificate_signature_scheme(u16::from_be_bytes([selector, split])); + let _cose_profile = RsaSignatureProfile::from_cose_algorithm_id(*select(&COSE_ALGORITHMS, selector)); + let _tls13_profile = RsaSignatureProfile::from_tls13_signature_scheme(u16::from_be_bytes([selector, split])); + let _certificate_profile = + RsaSignatureProfile::from_tls_certificate_signature_scheme(u16::from_be_bytes([selector, split])); } _ => { - let _ = RsaX509PublicKey::from_spki_der(message); + let _key_result = RsaX509PublicKey::from_spki_der(message); } } } diff --git a/fuzz/target_impls/auth_rsa_public_key.rs b/fuzz/target_impls/auth_rsa_public_key.rs index ddd6668f..d0d48e8c 100644 --- a/fuzz/target_impls/auth_rsa_public_key.rs +++ b/fuzz/target_impls/auth_rsa_public_key.rs @@ -1,7 +1,7 @@ use rscrypto::{RsaPublicKey, RsaPublicKeyPolicy}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let mode = some_or_return!(input.byte()); let raw_der = input.rest(); @@ -13,7 +13,7 @@ pub fn run(data: &[u8]) { raw_der }; - let parsed = match mode % 4 { + let parsed = match mode.rem_euclid(4) { 0 => RsaPublicKey::from_pkcs1_der(der), 1 => RsaPublicKey::from_spki_der(der), 2 => RsaPublicKey::from_pkcs1_der_with_policy(der, &RsaPublicKeyPolicy::modern_verification()), @@ -51,7 +51,7 @@ fn generated_public_key_der(mode: u8, control: &[u8]) -> Option> { Some(der) } Some(b'L') => Some(tlv_with_leading_zero_long_len(&public_key_der_for_mode(mode, &pkcs1))), - Some(b'U') if mode % 2 == 1 => Some(spki_public_key_der_with_unused_bits(&pkcs1)), + Some(b'U') if mode.rem_euclid(2) == 1 => Some(spki_public_key_der_with_unused_bits(&pkcs1)), Some(b'N') if mode.is_multiple_of(2) => Some(pkcs1_public_key_der_with_noncanonical_exponent()), _ => None, } @@ -122,7 +122,7 @@ fn algorithm_identifier(algorithm_oid: &[u8], params: Option<&[u8]>) -> Vec fn integer_unsigned(value: &[u8]) -> Vec { let first_nonzero = value.iter().position(|&byte| byte != 0); let value = first_nonzero.map_or(&[0u8][..], |index| &value[index..]); - let mut encoded = Vec::with_capacity(value.len() + usize::from(value[0] & 0x80 != 0)); + let mut encoded = Vec::with_capacity(value.len().strict_add(usize::from(value[0] & 0x80 != 0))); if value[0] & 0x80 != 0 { encoded.push(0); } @@ -131,7 +131,8 @@ fn integer_unsigned(value: &[u8]) -> Vec { } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + der_len(value.len()).len() + value.len()); + let encoded_len = 1usize.strict_add(der_len(value.len()).len()).strict_add(value.len()); + let mut out = Vec::with_capacity(encoded_len); out.push(tag); out.extend_from_slice(&der_len(value.len())); out.extend_from_slice(value); @@ -146,7 +147,8 @@ fn tlv_with_leading_zero_long_len(der: &[u8]) -> Vec { let mut out = Vec::with_capacity(der.len().strict_add(1)); out.push(tag); - out.push(0x80 | (len_len.strict_add(1) as u8)); + let encoded_len_len = u8::try_from(len_len.strict_add(1)).expect("DER length-of-length fits one byte"); + out.push(0x80 | encoded_len_len); out.push(0); out.extend_from_slice(&der[2..]); out @@ -154,14 +156,18 @@ fn tlv_with_leading_zero_long_len(der: &[u8]) -> Vec { fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("short-form DER length is below 128")]; } let bytes = len.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .expect("long-form DER length is nonzero"); let len_bytes = &bytes[first_nonzero..]; - let mut out = Vec::with_capacity(1 + len_bytes.len()); - out.push(0x80 | len_bytes.len() as u8); + let mut out = Vec::with_capacity(1usize.strict_add(len_bytes.len())); + let len_len = u8::try_from(len_bytes.len()).expect("usize DER length uses at most 16 bytes"); + out.push(0x80 | len_len); out.extend_from_slice(len_bytes); out } @@ -171,21 +177,21 @@ fn der_null() -> Vec { } fn hex_to_vec(hex: &str) -> Vec { - let mut out = Vec::with_capacity(hex.len() / 2); + let mut out = Vec::with_capacity(hex.len().div_euclid(2)); for chunk in hex.as_bytes().chunks_exact(2) { - let hi = hex_value(chunk[0]); - let lo = hex_value(chunk[1]); - out.push((hi << 4) | lo); + let hi = hex_value(chunk[0]).expect("RSA modulus fixture must contain hexadecimal digits"); + let lo = hex_value(chunk[1]).expect("RSA modulus fixture must contain hexadecimal digits"); + out.push(hi.strict_shl(4) | lo); } out } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => 0, + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } diff --git a/fuzz/target_impls/auth_rsa_verify.rs b/fuzz/target_impls/auth_rsa_verify.rs index 0fb1bd70..d4dadeb2 100644 --- a/fuzz/target_impls/auth_rsa_verify.rs +++ b/fuzz/target_impls/auth_rsa_verify.rs @@ -14,12 +14,12 @@ fn signature_candidate(material: &[u8], len: usize) -> Vec { } for (index, byte) in out.iter_mut().enumerate() { - *byte = material[index % material.len()]; + *byte = material[index.rem_euclid(material.len())]; } out } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let mode = some_or_return!(input.byte()); let split = some_or_return!(input.byte()); @@ -28,7 +28,7 @@ pub fn run(data: &[u8]) { let key = RsaPublicKey::from_spki_der(RSA3072_SPKI).expect("fuzz RSA fixture must parse"); let mut scratch = key.public_scratch(); - match mode % 6 { + match mode.rem_euclid(6) { 0 => { key .verify_pss_with_scratch(RsaPssProfile::Sha256, MESSAGE_PSS, RSA3072_PSS_SHA256, &mut scratch) @@ -46,17 +46,20 @@ pub fn run(data: &[u8]) { } 2 => { let signature = signature_candidate(signature_material, key.modulus().len()); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha256, message, &signature, &mut scratch); + let _verification_result = key.verify_pss_with_scratch(RsaPssProfile::Sha256, message, &signature, &mut scratch); } 3 => { let signature = signature_candidate(signature_material, key.modulus().len()); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, message, &signature, &mut scratch); + let _verification_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, message, &signature, &mut scratch); } 4 => { - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha384, message, signature_material, &mut scratch); + let _verification_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha384, message, signature_material, &mut scratch); } _ => { - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha512, message, signature_material, &mut scratch); + let _verification_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha512, message, signature_material, &mut scratch); } } } diff --git a/fuzz/target_impls/auth_scrypt.rs b/fuzz/target_impls/auth_scrypt.rs index 487dbfc9..bfba115b 100644 --- a/fuzz/target_impls/auth_scrypt.rs +++ b/fuzz/target_impls/auth_scrypt.rs @@ -1,7 +1,7 @@ use rscrypto::{Scrypt, ScryptParams}; use rscrypto_fuzz::{FuzzInput, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let pw_salt_split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); @@ -16,8 +16,7 @@ pub fn run(data: &[u8]) { let r = 1u32.strict_add(u32::from(r_byte) % 4); let out_len = 1u32.strict_add(u32::from(out_len_byte) % 64); - let params = ScryptParams::new(log_n, r, 1) - .expect("params must be valid for fuzzer ranges"); + let params = ScryptParams::new(log_n, r, 1).expect("params must be valid for fuzzer ranges"); let (password, salt) = split_at_ratio(rest, pw_salt_split); diff --git a/fuzz/target_impls/auth_x25519.rs b/fuzz/target_impls/auth_x25519.rs index d25e0521..a0c4ae17 100644 --- a/fuzz/target_impls/auth_x25519.rs +++ b/fuzz/target_impls/auth_x25519.rs @@ -1,7 +1,7 @@ use rscrypto::{X25519PublicKey, X25519SecretKey, X25519SharedSecret}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let secret_bytes: [u8; 32] = some_or_return!(input.bytes()); let peer_bytes: [u8; 32] = some_or_return!(input.bytes()); @@ -25,7 +25,11 @@ pub fn run(data: &[u8]) { match (&ours_shared, &helper_shared) { (Ok(ours), Ok(helper)) => assert!(ours.ct_eq(helper).declassify(), "x25519 helper mismatch"), (Err(_), Err(_)) => {} - _ => panic!("x25519 helper result mismatch"), + _ => assert_eq!( + ours_shared.is_ok(), + helper_shared.is_ok(), + "x25519 helper result mismatch" + ), } if dalek_shared.iter().all(|&byte| byte == 0) { diff --git a/fuzz/target_impls/checksum_crc.rs b/fuzz/target_impls/checksum_crc.rs index 2e116839..6ddde7c2 100644 --- a/fuzz/target_impls/checksum_crc.rs +++ b/fuzz/target_impls/checksum_crc.rs @@ -1,7 +1,7 @@ use rscrypto::{Checksum, ChecksumCombine, Crc16Ccitt, Crc16Ibm, Crc24OpenPgp, Crc32, Crc32C, Crc64, Crc64Nvme}; use rscrypto_fuzz::{FuzzInput, assert_checksum_chunked, assert_checksum_combine, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/checksum_crc16.rs b/fuzz/target_impls/checksum_crc16.rs index c6457f1b..1890bd6f 100644 --- a/fuzz/target_impls/checksum_crc16.rs +++ b/fuzz/target_impls/checksum_crc16.rs @@ -1,7 +1,7 @@ use rscrypto::{Checksum, Crc16Ccitt, Crc16Ibm}; use rscrypto_fuzz::{FuzzInput, assert_checksum_chunked, assert_checksum_combine, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/checksum_crc24.rs b/fuzz/target_impls/checksum_crc24.rs index 74161872..bccbca3b 100644 --- a/fuzz/target_impls/checksum_crc24.rs +++ b/fuzz/target_impls/checksum_crc24.rs @@ -1,7 +1,7 @@ use rscrypto::{Checksum, Crc24OpenPgp}; use rscrypto_fuzz::{FuzzInput, assert_checksum_chunked, assert_checksum_combine, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/checksum_crc32.rs b/fuzz/target_impls/checksum_crc32.rs index e91a65e3..edeaf35d 100644 --- a/fuzz/target_impls/checksum_crc32.rs +++ b/fuzz/target_impls/checksum_crc32.rs @@ -1,7 +1,7 @@ use rscrypto::{Checksum, Crc32, Crc32C}; use rscrypto_fuzz::{FuzzInput, assert_checksum_chunked, assert_checksum_combine, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/checksum_crc64.rs b/fuzz/target_impls/checksum_crc64.rs index 2bfe5a3c..69d7c223 100644 --- a/fuzz/target_impls/checksum_crc64.rs +++ b/fuzz/target_impls/checksum_crc64.rs @@ -1,7 +1,7 @@ use rscrypto::{Checksum, ChecksumCombine, Crc64, Crc64Nvme}; use rscrypto_fuzz::{FuzzInput, assert_checksum_chunked, assert_checksum_combine, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/fast_rapidhash.rs b/fuzz/target_impls/fast_rapidhash.rs index d3585184..c4e8bd3b 100644 --- a/fuzz/target_impls/fast_rapidhash.rs +++ b/fuzz/target_impls/fast_rapidhash.rs @@ -3,7 +3,7 @@ use core::hash::Hasher; use rscrypto::{RapidHash64, RapidStreamHasher}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let seed_bytes: [u8; 8] = some_or_return!(input.bytes()); let partitions: [u8; 8] = some_or_return!(input.bytes()); diff --git a/fuzz/target_impls/fast_xxh3.rs b/fuzz/target_impls/fast_xxh3.rs index 3a2c0cdf..12399a55 100644 --- a/fuzz/target_impls/fast_xxh3.rs +++ b/fuzz/target_impls/fast_xxh3.rs @@ -3,7 +3,7 @@ use core::hash::Hasher; use rscrypto::{FastHash, Xxh3, Xxh3_128, Xxh3_128Hasher, Xxh3Hasher}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let seed_bytes: [u8; 8] = some_or_return!(input.bytes()); let partitions: [u8; 8] = some_or_return!(input.bytes()); diff --git a/fuzz/target_impls/hash_ascon.rs b/fuzz/target_impls/hash_ascon.rs index 3529b687..c7cbdf80 100644 --- a/fuzz/target_impls/hash_ascon.rs +++ b/fuzz/target_impls/hash_ascon.rs @@ -1,7 +1,7 @@ use rscrypto::{AsconHash256, AsconXof, Digest, Xof}; use rscrypto_fuzz::{FuzzInput, assert_digest_chunked, assert_digest_reset, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); diff --git a/fuzz/target_impls/hash_ascon_cxof.rs b/fuzz/target_impls/hash_ascon_cxof.rs index 145e9a4e..a5bdbe88 100644 --- a/fuzz/target_impls/hash_ascon_cxof.rs +++ b/fuzz/target_impls/hash_ascon_cxof.rs @@ -18,7 +18,7 @@ fn extend_to_overlong(input: &[u8], control: u8) -> Vec { out } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let customization_split: u8 = some_or_return!(input.byte()); let message_split: u8 = some_or_return!(input.byte()); diff --git a/fuzz/target_impls/hash_blake2b.rs b/fuzz/target_impls/hash_blake2b.rs index 78266587..6e438e98 100644 --- a/fuzz/target_impls/hash_blake2b.rs +++ b/fuzz/target_impls/hash_blake2b.rs @@ -1,13 +1,18 @@ -use blake2::{Blake2b256 as OracleBlake2b256, Blake2b512 as OracleBlake2b512, Blake2bMac, Digest as _}; -use digest::typenum::{U32, U64}; -use hmac::{Mac as _, digest::KeyInit}; +use blake2::{ + Blake2b as OracleBlake2b, Blake2b512 as OracleBlake2b512, Blake2bMac, + digest::{ + Digest as _, Mac as _, + consts::{U32, U64}, + }, +}; use rscrypto::{Blake2b256, Blake2b512, Blake2bKey, Digest}; use rscrypto_fuzz::{FuzzInput, assert_digest_chunked, assert_digest_reset, some_or_return, split_at_ratio}; type OracleBlake2bMac256 = Blake2bMac; type OracleBlake2bMac512 = Blake2bMac; +type OracleBlake2b256 = OracleBlake2b; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let key_ratio: u8 = some_or_return!(input.byte()); @@ -30,7 +35,7 @@ pub fn run(data: &[u8]) { let split_idx = split_at_ratio(data, key_ratio).0.len(); let key_len = split_idx.clamp(1, 64); let (key, msg) = data.split_at(key_len); - let typed_key = Blake2bKey::new(key).unwrap(); + let typed_key = Blake2bKey::new(key).expect("the bounded BLAKE2b key length is valid"); let (msg_a, msg_b) = split_at_ratio(msg, split); let mut ours_256_stream = Blake2b256::new_keyed(typed_key); @@ -38,7 +43,8 @@ pub fn run(data: &[u8]) { ours_256_stream.update(msg_b); let ours_256_keyed = ours_256_stream.finalize(); - let mut oracle_256_mac = OracleBlake2bMac256::new_from_slice(key).unwrap(); + let mut oracle_256_mac = + OracleBlake2bMac256::new_from_slice(key).expect("the bounded BLAKE2b oracle key length is valid"); oracle_256_mac.update(msg); let oracle_256_keyed = oracle_256_mac.finalize().into_bytes(); assert_eq!(&ours_256_keyed[..], &oracle_256_keyed[..], "blake2b256 keyed mismatch"); @@ -48,7 +54,8 @@ pub fn run(data: &[u8]) { ours_512_stream.update(msg_b); let ours_512_keyed = ours_512_stream.finalize(); - let mut oracle_512_mac = OracleBlake2bMac512::new_from_slice(key).unwrap(); + let mut oracle_512_mac = + OracleBlake2bMac512::new_from_slice(key).expect("the bounded BLAKE2b oracle key length is valid"); oracle_512_mac.update(msg); let oracle_512_keyed = oracle_512_mac.finalize().into_bytes(); assert_eq!(&ours_512_keyed[..], &oracle_512_keyed[..], "blake2b512 keyed mismatch"); diff --git a/fuzz/target_impls/hash_blake2s.rs b/fuzz/target_impls/hash_blake2s.rs index 8446d3c6..22215ae8 100644 --- a/fuzz/target_impls/hash_blake2s.rs +++ b/fuzz/target_impls/hash_blake2s.rs @@ -1,13 +1,18 @@ -use blake2::{Blake2s128 as OracleBlake2s128, Blake2s256 as OracleBlake2s256, Blake2sMac, Digest as _}; -use digest::typenum::{U16, U32}; -use hmac::{Mac as _, digest::KeyInit}; +use blake2::{ + Blake2s as OracleBlake2s, Blake2s256 as OracleBlake2s256, Blake2sMac, + digest::{ + Digest as _, Mac as _, + consts::{U16, U32}, + }, +}; use rscrypto::{Blake2s128, Blake2s256, Blake2sKey, Digest}; use rscrypto_fuzz::{FuzzInput, assert_digest_chunked, assert_digest_reset, some_or_return, split_at_ratio}; type OracleBlake2sMac128 = Blake2sMac; type OracleBlake2sMac256 = Blake2sMac; +type OracleBlake2s128 = OracleBlake2s; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let key_ratio: u8 = some_or_return!(input.byte()); @@ -30,7 +35,7 @@ pub fn run(data: &[u8]) { let split_idx = split_at_ratio(data, key_ratio).0.len(); let key_len = split_idx.clamp(1, 32); let (key, msg) = data.split_at(key_len); - let typed_key = Blake2sKey::new(key).unwrap(); + let typed_key = Blake2sKey::new(key).expect("the bounded BLAKE2s key length is valid"); let (msg_a, msg_b) = split_at_ratio(msg, split); let mut ours_128_stream = Blake2s128::new_keyed(typed_key); @@ -38,7 +43,8 @@ pub fn run(data: &[u8]) { ours_128_stream.update(msg_b); let ours_128_keyed = ours_128_stream.finalize(); - let mut oracle_128_mac = OracleBlake2sMac128::new_from_slice(key).unwrap(); + let mut oracle_128_mac = + OracleBlake2sMac128::new_from_slice(key).expect("the bounded BLAKE2s oracle key length is valid"); oracle_128_mac.update(msg); let oracle_128_keyed = oracle_128_mac.finalize().into_bytes(); assert_eq!(&ours_128_keyed[..], &oracle_128_keyed[..], "blake2s128 keyed mismatch"); @@ -48,7 +54,8 @@ pub fn run(data: &[u8]) { ours_256_stream.update(msg_b); let ours_256_keyed = ours_256_stream.finalize(); - let mut oracle_256_mac = OracleBlake2sMac256::new_from_slice(key).unwrap(); + let mut oracle_256_mac = + OracleBlake2sMac256::new_from_slice(key).expect("the bounded BLAKE2s oracle key length is valid"); oracle_256_mac.update(msg); let oracle_256_keyed = oracle_256_mac.finalize().into_bytes(); assert_eq!(&ours_256_keyed[..], &oracle_256_keyed[..], "blake2s256 keyed mismatch"); diff --git a/fuzz/target_impls/hash_blake3.rs b/fuzz/target_impls/hash_blake3.rs index 9878e469..888c9caa 100644 --- a/fuzz/target_impls/hash_blake3.rs +++ b/fuzz/target_impls/hash_blake3.rs @@ -1,7 +1,7 @@ use rscrypto::{Blake3, Digest}; use rscrypto_fuzz::{FuzzInput, assert_digest_chunked, assert_digest_reset, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); diff --git a/fuzz/target_impls/hash_blake3_derive.rs b/fuzz/target_impls/hash_blake3_derive.rs index 36aaf65b..cb1d318c 100644 --- a/fuzz/target_impls/hash_blake3_derive.rs +++ b/fuzz/target_impls/hash_blake3_derive.rs @@ -6,7 +6,7 @@ use rscrypto_fuzz::{FuzzInput, assert_xof_prefix, some_or_return}; // separator, not the interesting input. const CONTEXT: &str = "rscrypto fuzz 2026-04-12 derive_key test context"; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/hash_blake3_keyed.rs b/fuzz/target_impls/hash_blake3_keyed.rs index 6da3b9ee..dacca9bc 100644 --- a/fuzz/target_impls/hash_blake3_keyed.rs +++ b/fuzz/target_impls/hash_blake3_keyed.rs @@ -1,7 +1,7 @@ use rscrypto::{Blake3, Blake3KeyedHash, Digest}; use rscrypto_fuzz::{FuzzInput, assert_xof_prefix, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key: [u8; 32] = some_or_return!(input.bytes()); let split: u8 = some_or_return!(input.byte()); diff --git a/fuzz/target_impls/hash_cshake256.rs b/fuzz/target_impls/hash_cshake256.rs index 5ebdc605..f03b421f 100644 --- a/fuzz/target_impls/hash_cshake256.rs +++ b/fuzz/target_impls/hash_cshake256.rs @@ -1,7 +1,7 @@ use rscrypto::{Cshake256, Xof}; use rscrypto_fuzz::{FuzzInput, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let name_split: u8 = some_or_return!(input.byte()); let custom_split: u8 = some_or_return!(input.byte()); diff --git a/fuzz/target_impls/hash_sha2.rs b/fuzz/target_impls/hash_sha2.rs index 2dfcc9e2..b0e3be79 100644 --- a/fuzz/target_impls/hash_sha2.rs +++ b/fuzz/target_impls/hash_sha2.rs @@ -1,7 +1,7 @@ use rscrypto::{Sha224, Sha256, Sha384, Sha512, Sha512_256}; use rscrypto_fuzz::{FuzzInput, assert_digest_chunked, assert_digest_reset, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/hash_sha3.rs b/fuzz/target_impls/hash_sha3.rs index 2082a539..78dd50a6 100644 --- a/fuzz/target_impls/hash_sha3.rs +++ b/fuzz/target_impls/hash_sha3.rs @@ -1,7 +1,7 @@ use rscrypto::{Digest, Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256, Xof}; use rscrypto_fuzz::{FuzzInput, assert_digest_chunked, assert_digest_reset, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); diff --git a/fuzz/target_impls/hex_parse.rs b/fuzz/target_impls/hex_parse.rs index 44a9e120..4eb36930 100644 --- a/fuzz/target_impls/hex_parse.rs +++ b/fuzz/target_impls/hex_parse.rs @@ -23,7 +23,7 @@ fn ascii_candidate(bytes: &[u8], len: usize) -> String { .copied() .cycle() .take(len) - .map(|byte| char::from(32 + (byte % 95))) + .map(|byte| char::from(byte.rem_euclid(95).strict_add(32))) .collect() } @@ -33,10 +33,20 @@ where { let lower = value.to_string(); let upper = format!("{value:X}"); - let short = &lower[..lower.len().strict_sub(1)]; + let short = lower + .get(..lower.len().strict_sub(1)) + .expect("hex rendering has an ASCII boundary before its final byte"); - assert_eq!(lower.parse::().unwrap(), value, "public lower parse mismatch"); - assert_eq!(upper.parse::().unwrap(), value, "public upper parse mismatch"); + assert_eq!( + lower.parse::().expect("lowercase rendering must parse"), + value, + "public lower parse mismatch" + ); + assert_eq!( + upper.parse::().expect("uppercase rendering must parse"), + value, + "public upper parse mismatch" + ); assert_eq!( short.parse::(), Err(InvalidHexError::InvalidLength), @@ -44,8 +54,19 @@ where ); if let Ok(parsed) = candidate.parse::() { - assert_eq!(parsed.to_string().parse::().unwrap(), parsed); - assert_eq!(format!("{parsed:X}").parse::().unwrap(), parsed); + assert_eq!( + parsed + .to_string() + .parse::() + .expect("parsed value must round-trip through lowercase rendering"), + parsed + ); + assert_eq!( + format!("{parsed:X}") + .parse::() + .expect("parsed value must round-trip through uppercase rendering"), + parsed + ); } } @@ -58,14 +79,22 @@ fn exercise_sensitive_parse( ) where T: Debug + FromStr, { - let short = &lower[..lower.len().strict_sub(1)]; + let short = lower + .get(..lower.len().strict_sub(1)) + .expect("hex rendering has an ASCII boundary before its final byte"); assert!( - equivalent(&lower.parse::().unwrap(), &value), + equivalent( + &lower.parse::().expect("lowercase secret rendering must parse"), + &value + ), "sensitive lower parse mismatch" ); assert!( - equivalent(&upper.parse::().unwrap(), &value), + equivalent( + &upper.parse::().expect("uppercase secret rendering must parse"), + &value + ), "sensitive upper parse mismatch" ); assert!( @@ -79,7 +108,7 @@ fn exercise_sensitive_parse( } } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let nonce_bytes: [u8; 12] = some_or_return!(input.bytes()); let tag_bytes: [u8; 16] = some_or_return!(input.bytes()); diff --git a/fuzz/target_impls/traits_io.rs b/fuzz/target_impls/traits_io.rs index 786e6f7c..02400efa 100644 --- a/fuzz/target_impls/traits_io.rs +++ b/fuzz/target_impls/traits_io.rs @@ -44,7 +44,7 @@ fn sequential_chunks(data: &[u8], lens: [usize; 3]) -> [&[u8]; 3] { } fn reader_step(reader: &mut R, seen: &mut Vec, control: u8) -> io::Result<()> { - match control % 3 { + match control.rem_euclid(3) { 0 => { let mut buf = vec![0u8; scalar_len(control)]; let n = reader.read(&mut buf)?; @@ -68,7 +68,9 @@ fn reader_step(reader: &mut R, seen: &mut Vec, control: u8) -> io:: } _ => { let n = reader.read(&mut [])?; - assert_eq!(n, 0, "zero-length read must report zero"); + if n != 0 { + return Err(io::Error::other("zero-length read returned a nonzero length")); + } } } @@ -82,7 +84,7 @@ fn writer_step( accepted: &mut Vec, control: u8, ) -> io::Result<()> { - match control % 4 { + match control.rem_euclid(4) { 0 => { let request = scalar_len(control).min(data.len().strict_sub(*cursor)); let end = cursor.strict_add(request); @@ -103,7 +105,9 @@ fn writer_step( } 2 => { let n = writer.write(&[])?; - assert_eq!(n, 0, "zero-length write must report zero"); + if n != 0 { + return Err(io::Error::other("zero-length write returned a nonzero length")); + } } _ => writer.flush()?, } @@ -275,7 +279,7 @@ fn fuzz_digest_writer(data: &[u8], ops: &[u8], max_per_call: usize) { assert_eq!(digest, Blake3::digest(data), "digest writer final mismatch"); } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let reader_limit = usize::from(some_or_return!(input.byte()) % 32).strict_add(1); let writer_limit = usize::from(some_or_return!(input.byte()) % 32).strict_add(1); diff --git a/fuzz/tests/corpus_replay.rs b/fuzz/tests/corpus_replay.rs index ad1ff5dc..c9152356 100644 --- a/fuzz/tests/corpus_replay.rs +++ b/fuzz/tests/corpus_replay.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use rscrypto_fuzz::replay_corpus_dir; fn corpus_dir(target: &str) -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("corpus").join(target) + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("corpus").join(target) } #[path = "../target_impls/aead_aegis256.rs"] @@ -149,282 +149,346 @@ mod traits_io; #[test] fn replay_aead_aegis256_corpus() { - let replayed = replay_corpus_dir("aead_aegis256", corpus_dir("aead_aegis256"), aead_aegis256::run); - assert_ne!(replayed, 0, "aead_aegis256 corpus should not be empty"); + let replayed = replay_corpus_dir("aead_aegis256", corpus_dir("aead_aegis256"), aead_aegis256::run); + assert_ne!(replayed, 0, "aead_aegis256 corpus should not be empty"); } #[test] fn replay_aead_aes256gcm_corpus() { - let replayed = replay_corpus_dir("aead_aes256gcm", corpus_dir("aead_aes256gcm"), aead_aes256gcm::run); - assert_ne!(replayed, 0, "aead_aes256gcm corpus should not be empty"); + let replayed = replay_corpus_dir("aead_aes256gcm", corpus_dir("aead_aes256gcm"), aead_aes256gcm::run); + assert_ne!(replayed, 0, "aead_aes256gcm corpus should not be empty"); } #[test] fn replay_aead_aes256gcmsiv_corpus() { - let replayed = replay_corpus_dir("aead_aes256gcmsiv", corpus_dir("aead_aes256gcmsiv"), aead_aes256gcmsiv::run); - assert_ne!(replayed, 0, "aead_aes256gcmsiv corpus should not be empty"); + let replayed = replay_corpus_dir( + "aead_aes256gcmsiv", + corpus_dir("aead_aes256gcmsiv"), + aead_aes256gcmsiv::run, + ); + assert_ne!(replayed, 0, "aead_aes256gcmsiv corpus should not be empty"); } #[test] fn replay_aead_ascon128_corpus() { - let replayed = replay_corpus_dir("aead_ascon128", corpus_dir("aead_ascon128"), aead_ascon128::run); - assert_ne!(replayed, 0, "aead_ascon128 corpus should not be empty"); + let replayed = replay_corpus_dir("aead_ascon128", corpus_dir("aead_ascon128"), aead_ascon128::run); + assert_ne!(replayed, 0, "aead_ascon128 corpus should not be empty"); } #[test] fn replay_aead_chacha20poly1305_corpus() { - let replayed = replay_corpus_dir("aead_chacha20poly1305", corpus_dir("aead_chacha20poly1305"), aead_chacha20poly1305::run); - assert_ne!(replayed, 0, "aead_chacha20poly1305 corpus should not be empty"); + let replayed = replay_corpus_dir( + "aead_chacha20poly1305", + corpus_dir("aead_chacha20poly1305"), + aead_chacha20poly1305::run, + ); + assert_ne!(replayed, 0, "aead_chacha20poly1305 corpus should not be empty"); } #[test] fn replay_aead_nonce_counter_corpus() { - let replayed = replay_corpus_dir("aead_nonce_counter", corpus_dir("aead_nonce_counter"), aead_nonce_counter::run); - assert_ne!(replayed, 0, "aead_nonce_counter corpus should not be empty"); + let replayed = replay_corpus_dir( + "aead_nonce_counter", + corpus_dir("aead_nonce_counter"), + aead_nonce_counter::run, + ); + assert_ne!(replayed, 0, "aead_nonce_counter corpus should not be empty"); } #[test] fn replay_aead_xchacha20poly1305_corpus() { - let replayed = replay_corpus_dir("aead_xchacha20poly1305", corpus_dir("aead_xchacha20poly1305"), aead_xchacha20poly1305::run); - assert_ne!(replayed, 0, "aead_xchacha20poly1305 corpus should not be empty"); + let replayed = replay_corpus_dir( + "aead_xchacha20poly1305", + corpus_dir("aead_xchacha20poly1305"), + aead_xchacha20poly1305::run, + ); + assert_ne!(replayed, 0, "aead_xchacha20poly1305 corpus should not be empty"); } #[test] fn replay_auth_argon2d_corpus() { - let replayed = replay_corpus_dir("auth_argon2d", corpus_dir("auth_argon2d"), auth_argon2d::run); - assert_ne!(replayed, 0, "auth_argon2d corpus should not be empty"); + let replayed = replay_corpus_dir("auth_argon2d", corpus_dir("auth_argon2d"), auth_argon2d::run); + assert_ne!(replayed, 0, "auth_argon2d corpus should not be empty"); } #[test] fn replay_auth_argon2i_corpus() { - let replayed = replay_corpus_dir("auth_argon2i", corpus_dir("auth_argon2i"), auth_argon2i::run); - assert_ne!(replayed, 0, "auth_argon2i corpus should not be empty"); + let replayed = replay_corpus_dir("auth_argon2i", corpus_dir("auth_argon2i"), auth_argon2i::run); + assert_ne!(replayed, 0, "auth_argon2i corpus should not be empty"); } #[test] fn replay_auth_argon2id_corpus() { - let replayed = replay_corpus_dir("auth_argon2id", corpus_dir("auth_argon2id"), auth_argon2id::run); - assert_ne!(replayed, 0, "auth_argon2id corpus should not be empty"); + let replayed = replay_corpus_dir("auth_argon2id", corpus_dir("auth_argon2id"), auth_argon2id::run); + assert_ne!(replayed, 0, "auth_argon2id corpus should not be empty"); } #[test] fn replay_auth_ed25519_corpus() { - let replayed = replay_corpus_dir("auth_ed25519", corpus_dir("auth_ed25519"), auth_ed25519::run); - assert_ne!(replayed, 0, "auth_ed25519 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_ed25519", corpus_dir("auth_ed25519"), auth_ed25519::run); + assert_ne!(replayed, 0, "auth_ed25519 corpus should not be empty"); } #[test] fn replay_auth_ed25519_verify_corpus() { - let replayed = replay_corpus_dir("auth_ed25519_verify", corpus_dir("auth_ed25519_verify"), auth_ed25519_verify::run); - assert_ne!(replayed, 0, "auth_ed25519_verify corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_ed25519_verify", + corpus_dir("auth_ed25519_verify"), + auth_ed25519_verify::run, + ); + assert_ne!(replayed, 0, "auth_ed25519_verify corpus should not be empty"); } #[test] fn replay_auth_ecdsa_verify_corpus() { - let replayed = replay_corpus_dir("auth_ecdsa_verify", corpus_dir("auth_ecdsa_verify"), auth_ecdsa_verify::run); - assert_ne!(replayed, 0, "auth_ecdsa_verify corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_ecdsa_verify", + corpus_dir("auth_ecdsa_verify"), + auth_ecdsa_verify::run, + ); + assert_ne!(replayed, 0, "auth_ecdsa_verify corpus should not be empty"); } #[test] fn replay_auth_ecdsa_sign_corpus() { - let replayed = replay_corpus_dir("auth_ecdsa_sign", corpus_dir("auth_ecdsa_sign"), auth_ecdsa_sign::run); - assert_ne!(replayed, 0, "auth_ecdsa_sign corpus should not be empty"); + let replayed = replay_corpus_dir("auth_ecdsa_sign", corpus_dir("auth_ecdsa_sign"), auth_ecdsa_sign::run); + assert_ne!(replayed, 0, "auth_ecdsa_sign corpus should not be empty"); } #[test] fn replay_auth_hkdf_sha256_corpus() { - let replayed = replay_corpus_dir("auth_hkdf_sha256", corpus_dir("auth_hkdf_sha256"), auth_hkdf_sha256::run); - assert_ne!(replayed, 0, "auth_hkdf_sha256 corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_hkdf_sha256", + corpus_dir("auth_hkdf_sha256"), + auth_hkdf_sha256::run, + ); + assert_ne!(replayed, 0, "auth_hkdf_sha256 corpus should not be empty"); } #[test] fn replay_auth_hkdf_sha384_corpus() { - let replayed = replay_corpus_dir("auth_hkdf_sha384", corpus_dir("auth_hkdf_sha384"), auth_hkdf_sha384::run); - assert_ne!(replayed, 0, "auth_hkdf_sha384 corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_hkdf_sha384", + corpus_dir("auth_hkdf_sha384"), + auth_hkdf_sha384::run, + ); + assert_ne!(replayed, 0, "auth_hkdf_sha384 corpus should not be empty"); } #[test] fn replay_auth_hmac_sha256_corpus() { - let replayed = replay_corpus_dir("auth_hmac_sha256", corpus_dir("auth_hmac_sha256"), auth_hmac_sha256::run); - assert_ne!(replayed, 0, "auth_hmac_sha256 corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_hmac_sha256", + corpus_dir("auth_hmac_sha256"), + auth_hmac_sha256::run, + ); + assert_ne!(replayed, 0, "auth_hmac_sha256 corpus should not be empty"); } #[test] fn replay_auth_hmac_sha384_corpus() { - let replayed = replay_corpus_dir("auth_hmac_sha384", corpus_dir("auth_hmac_sha384"), auth_hmac_sha384::run); - assert_ne!(replayed, 0, "auth_hmac_sha384 corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_hmac_sha384", + corpus_dir("auth_hmac_sha384"), + auth_hmac_sha384::run, + ); + assert_ne!(replayed, 0, "auth_hmac_sha384 corpus should not be empty"); } #[test] fn replay_auth_hmac_sha512_corpus() { - let replayed = replay_corpus_dir("auth_hmac_sha512", corpus_dir("auth_hmac_sha512"), auth_hmac_sha512::run); - assert_ne!(replayed, 0, "auth_hmac_sha512 corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_hmac_sha512", + corpus_dir("auth_hmac_sha512"), + auth_hmac_sha512::run, + ); + assert_ne!(replayed, 0, "auth_hmac_sha512 corpus should not be empty"); } #[test] fn replay_auth_kmac256_corpus() { - let replayed = replay_corpus_dir("auth_kmac256", corpus_dir("auth_kmac256"), auth_kmac256::run); - assert_ne!(replayed, 0, "auth_kmac256 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_kmac256", corpus_dir("auth_kmac256"), auth_kmac256::run); + assert_ne!(replayed, 0, "auth_kmac256 corpus should not be empty"); } #[test] fn replay_auth_mlkem512_corpus() { - let replayed = replay_corpus_dir("auth_mlkem512", corpus_dir("auth_mlkem512"), auth_mlkem512::run); - assert_ne!(replayed, 0, "auth_mlkem512 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_mlkem512", corpus_dir("auth_mlkem512"), auth_mlkem512::run); + assert_ne!(replayed, 0, "auth_mlkem512 corpus should not be empty"); } #[test] fn replay_auth_mlkem768_corpus() { - let replayed = replay_corpus_dir("auth_mlkem768", corpus_dir("auth_mlkem768"), auth_mlkem768::run); - assert_ne!(replayed, 0, "auth_mlkem768 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_mlkem768", corpus_dir("auth_mlkem768"), auth_mlkem768::run); + assert_ne!(replayed, 0, "auth_mlkem768 corpus should not be empty"); } #[test] fn replay_auth_mlkem1024_corpus() { - let replayed = replay_corpus_dir("auth_mlkem1024", corpus_dir("auth_mlkem1024"), auth_mlkem1024::run); - assert_ne!(replayed, 0, "auth_mlkem1024 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_mlkem1024", corpus_dir("auth_mlkem1024"), auth_mlkem1024::run); + assert_ne!(replayed, 0, "auth_mlkem1024 corpus should not be empty"); } #[test] fn replay_auth_pbkdf2_corpus() { - let replayed = replay_corpus_dir("auth_pbkdf2", corpus_dir("auth_pbkdf2"), auth_pbkdf2::run); - assert_ne!(replayed, 0, "auth_pbkdf2 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_pbkdf2", corpus_dir("auth_pbkdf2"), auth_pbkdf2::run); + assert_ne!(replayed, 0, "auth_pbkdf2 corpus should not be empty"); } #[test] fn replay_auth_phc_corpus() { - let replayed = replay_corpus_dir("auth_phc", corpus_dir("auth_phc"), auth_phc::run); - assert_ne!(replayed, 0, "auth_phc corpus should not be empty"); + let replayed = replay_corpus_dir("auth_phc", corpus_dir("auth_phc"), auth_phc::run); + assert_ne!(replayed, 0, "auth_phc corpus should not be empty"); } #[test] fn replay_auth_rsa_public_key_corpus() { - let replayed = replay_corpus_dir("auth_rsa_public_key", corpus_dir("auth_rsa_public_key"), auth_rsa_public_key::run); - assert_ne!(replayed, 0, "auth_rsa_public_key corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_rsa_public_key", + corpus_dir("auth_rsa_public_key"), + auth_rsa_public_key::run, + ); + assert_ne!(replayed, 0, "auth_rsa_public_key corpus should not be empty"); } #[test] fn replay_auth_rsa_import_corpus() { - let replayed = replay_corpus_dir("auth_rsa_import", corpus_dir("auth_rsa_import"), auth_rsa_import::run); - assert_ne!(replayed, 0, "auth_rsa_import corpus should not be empty"); + let replayed = replay_corpus_dir("auth_rsa_import", corpus_dir("auth_rsa_import"), auth_rsa_import::run); + assert_ne!(replayed, 0, "auth_rsa_import corpus should not be empty"); } #[test] fn replay_auth_rsa_protocol_corpus() { - let replayed = replay_corpus_dir("auth_rsa_protocol", corpus_dir("auth_rsa_protocol"), auth_rsa_protocol::run); - assert_ne!(replayed, 0, "auth_rsa_protocol corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_rsa_protocol", + corpus_dir("auth_rsa_protocol"), + auth_rsa_protocol::run, + ); + assert_ne!(replayed, 0, "auth_rsa_protocol corpus should not be empty"); } #[test] fn replay_auth_rsa_private_ops_corpus() { - let replayed = replay_corpus_dir("auth_rsa_private_ops", corpus_dir("auth_rsa_private_ops"), auth_rsa_private_ops::run); - assert_ne!(replayed, 0, "auth_rsa_private_ops corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_rsa_private_ops", + corpus_dir("auth_rsa_private_ops"), + auth_rsa_private_ops::run, + ); + assert_ne!(replayed, 0, "auth_rsa_private_ops corpus should not be empty"); } #[test] fn replay_auth_rsa_verify_corpus() { - let replayed = replay_corpus_dir("auth_rsa_verify", corpus_dir("auth_rsa_verify"), auth_rsa_verify::run); - assert_ne!(replayed, 0, "auth_rsa_verify corpus should not be empty"); + let replayed = replay_corpus_dir("auth_rsa_verify", corpus_dir("auth_rsa_verify"), auth_rsa_verify::run); + assert_ne!(replayed, 0, "auth_rsa_verify corpus should not be empty"); } #[test] fn replay_auth_scrypt_corpus() { - let replayed = replay_corpus_dir("auth_scrypt", corpus_dir("auth_scrypt"), auth_scrypt::run); - assert_ne!(replayed, 0, "auth_scrypt corpus should not be empty"); + let replayed = replay_corpus_dir("auth_scrypt", corpus_dir("auth_scrypt"), auth_scrypt::run); + assert_ne!(replayed, 0, "auth_scrypt corpus should not be empty"); } #[test] fn replay_auth_x25519_corpus() { - let replayed = replay_corpus_dir("auth_x25519", corpus_dir("auth_x25519"), auth_x25519::run); - assert_ne!(replayed, 0, "auth_x25519 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_x25519", corpus_dir("auth_x25519"), auth_x25519::run); + assert_ne!(replayed, 0, "auth_x25519 corpus should not be empty"); } #[test] fn replay_checksum_crc_corpus() { - let replayed = replay_corpus_dir("checksum_crc", corpus_dir("checksum_crc"), checksum_crc::run); - assert_ne!(replayed, 0, "checksum_crc corpus should not be empty"); + let replayed = replay_corpus_dir("checksum_crc", corpus_dir("checksum_crc"), checksum_crc::run); + assert_ne!(replayed, 0, "checksum_crc corpus should not be empty"); } #[test] fn replay_fast_rapidhash_corpus() { - let replayed = replay_corpus_dir("fast_rapidhash", corpus_dir("fast_rapidhash"), fast_rapidhash::run); - assert_ne!(replayed, 0, "fast_rapidhash corpus should not be empty"); + let replayed = replay_corpus_dir("fast_rapidhash", corpus_dir("fast_rapidhash"), fast_rapidhash::run); + assert_ne!(replayed, 0, "fast_rapidhash corpus should not be empty"); } #[test] fn replay_fast_xxh3_corpus() { - let replayed = replay_corpus_dir("fast_xxh3", corpus_dir("fast_xxh3"), fast_xxh3::run); - assert_ne!(replayed, 0, "fast_xxh3 corpus should not be empty"); + let replayed = replay_corpus_dir("fast_xxh3", corpus_dir("fast_xxh3"), fast_xxh3::run); + assert_ne!(replayed, 0, "fast_xxh3 corpus should not be empty"); } #[test] fn replay_hash_ascon_corpus() { - let replayed = replay_corpus_dir("hash_ascon", corpus_dir("hash_ascon"), hash_ascon::run); - assert_ne!(replayed, 0, "hash_ascon corpus should not be empty"); + let replayed = replay_corpus_dir("hash_ascon", corpus_dir("hash_ascon"), hash_ascon::run); + assert_ne!(replayed, 0, "hash_ascon corpus should not be empty"); } #[test] fn replay_hash_ascon_cxof_corpus() { - let replayed = replay_corpus_dir("hash_ascon_cxof", corpus_dir("hash_ascon_cxof"), hash_ascon_cxof::run); - assert_ne!(replayed, 0, "hash_ascon_cxof corpus should not be empty"); + let replayed = replay_corpus_dir("hash_ascon_cxof", corpus_dir("hash_ascon_cxof"), hash_ascon_cxof::run); + assert_ne!(replayed, 0, "hash_ascon_cxof corpus should not be empty"); } #[test] fn replay_hash_blake2b_corpus() { - let replayed = replay_corpus_dir("hash_blake2b", corpus_dir("hash_blake2b"), hash_blake2b::run); - assert_ne!(replayed, 0, "hash_blake2b corpus should not be empty"); + let replayed = replay_corpus_dir("hash_blake2b", corpus_dir("hash_blake2b"), hash_blake2b::run); + assert_ne!(replayed, 0, "hash_blake2b corpus should not be empty"); } #[test] fn replay_hash_blake2s_corpus() { - let replayed = replay_corpus_dir("hash_blake2s", corpus_dir("hash_blake2s"), hash_blake2s::run); - assert_ne!(replayed, 0, "hash_blake2s corpus should not be empty"); + let replayed = replay_corpus_dir("hash_blake2s", corpus_dir("hash_blake2s"), hash_blake2s::run); + assert_ne!(replayed, 0, "hash_blake2s corpus should not be empty"); } #[test] fn replay_hash_blake3_corpus() { - let replayed = replay_corpus_dir("hash_blake3", corpus_dir("hash_blake3"), hash_blake3::run); - assert_ne!(replayed, 0, "hash_blake3 corpus should not be empty"); + let replayed = replay_corpus_dir("hash_blake3", corpus_dir("hash_blake3"), hash_blake3::run); + assert_ne!(replayed, 0, "hash_blake3 corpus should not be empty"); } #[test] fn replay_hash_blake3_derive_corpus() { - let replayed = replay_corpus_dir("hash_blake3_derive", corpus_dir("hash_blake3_derive"), hash_blake3_derive::run); - assert_ne!(replayed, 0, "hash_blake3_derive corpus should not be empty"); + let replayed = replay_corpus_dir( + "hash_blake3_derive", + corpus_dir("hash_blake3_derive"), + hash_blake3_derive::run, + ); + assert_ne!(replayed, 0, "hash_blake3_derive corpus should not be empty"); } #[test] fn replay_hash_blake3_keyed_corpus() { - let replayed = replay_corpus_dir("hash_blake3_keyed", corpus_dir("hash_blake3_keyed"), hash_blake3_keyed::run); - assert_ne!(replayed, 0, "hash_blake3_keyed corpus should not be empty"); + let replayed = replay_corpus_dir( + "hash_blake3_keyed", + corpus_dir("hash_blake3_keyed"), + hash_blake3_keyed::run, + ); + assert_ne!(replayed, 0, "hash_blake3_keyed corpus should not be empty"); } #[test] fn replay_hash_cshake256_corpus() { - let replayed = replay_corpus_dir("hash_cshake256", corpus_dir("hash_cshake256"), hash_cshake256::run); - assert_ne!(replayed, 0, "hash_cshake256 corpus should not be empty"); + let replayed = replay_corpus_dir("hash_cshake256", corpus_dir("hash_cshake256"), hash_cshake256::run); + assert_ne!(replayed, 0, "hash_cshake256 corpus should not be empty"); } #[test] fn replay_hash_sha2_corpus() { - let replayed = replay_corpus_dir("hash_sha2", corpus_dir("hash_sha2"), hash_sha2::run); - assert_ne!(replayed, 0, "hash_sha2 corpus should not be empty"); + let replayed = replay_corpus_dir("hash_sha2", corpus_dir("hash_sha2"), hash_sha2::run); + assert_ne!(replayed, 0, "hash_sha2 corpus should not be empty"); } #[test] fn replay_hash_sha3_corpus() { - let replayed = replay_corpus_dir("hash_sha3", corpus_dir("hash_sha3"), hash_sha3::run); - assert_ne!(replayed, 0, "hash_sha3 corpus should not be empty"); + let replayed = replay_corpus_dir("hash_sha3", corpus_dir("hash_sha3"), hash_sha3::run); + assert_ne!(replayed, 0, "hash_sha3 corpus should not be empty"); } #[test] fn replay_hex_parse_corpus() { - let replayed = replay_corpus_dir("hex_parse", corpus_dir("hex_parse"), hex_parse::run); - assert_ne!(replayed, 0, "hex_parse corpus should not be empty"); + let replayed = replay_corpus_dir("hex_parse", corpus_dir("hex_parse"), hex_parse::run); + assert_ne!(replayed, 0, "hex_parse corpus should not be empty"); } #[test] fn replay_traits_io_corpus() { - let replayed = replay_corpus_dir("traits_io", corpus_dir("traits_io"), traits_io::run); - assert_ne!(replayed, 0, "traits_io corpus should not be empty"); + let replayed = replay_corpus_dir("traits_io", corpus_dir("traits_io"), traits_io::run); + assert_ne!(replayed, 0, "traits_io corpus should not be empty"); } diff --git a/justfile b/justfile index 3da40268..a2dc6e0d 100644 --- a/justfile +++ b/justfile @@ -1,104 +1,49 @@ # Remote dev. Provider mechanics live in ~/dev-machines. -ssh target: - @"$HOME/dev-machines/dev-machine" ssh rscrypto "{{ target }}" +dev_machine := env_var_or_default("DEV_MACHINE_BIN", env_var("HOME") + "/dev-machines/dev-machine") -ssh-check target: - @"$HOME/dev-machines/dev-machine" ssh rscrypto "{{ target }}" --check +ssh target *args="": + @"{{ dev_machine }}" ssh rscrypto "{{ target }}" {{ args }} -ssh-create target *args="": - @"$HOME/dev-machines/dev-machine" create rscrypto "{{ target }}" {{ args }} - -ssh-kill target: - @"$HOME/dev-machines/dev-machine" kill rscrypto "{{ target }}" - -ssh-status target="": - @if [ -n "{{ target }}" ]; then "$HOME/dev-machines/dev-machine" status rscrypto "{{ target }}"; else "$HOME/dev-machines/dev-machine" status rscrypto; fi - -ssh-bootstrap target: - @"$HOME/dev-machines/dev-machine" bootstrap rscrypto "{{ target }}" - -ssh-aws-linux-x64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-x64 - -ssh-aws-linux-arm64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-arm64 - -ssh-aws-windows-x64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-windows-x64 - -ssh-azure-linux-x64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-x64 - -ssh-azure-linux-arm64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-arm64 - -ssh-azure-windows-x64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-windows-x64 +ssh-check target *args="": + @"{{ dev_machine }}" ssh rscrypto "{{ target }}" --check {{ args }} -ssh-azure-windows-arm64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-windows-arm64 +ssh-preflight target: + @"{{ dev_machine }}" preflight rscrypto "{{ target }}" -ssh-aws-linux-x64-perf: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-x64-perf - -ssh-aws-linux-intel-gnr-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-intel-gnr-profile - -ssh-aws-linux-intel-spr-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-intel-spr-profile - -ssh-aws-linux-amd-zen5-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-amd-zen5-profile - -ssh-aws-linux-amd-zen4-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-amd-zen4-profile - -ssh-aws-linux-arm64-graviton3-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-arm64-graviton3-profile - -ssh-aws-linux-arm64-graviton4-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-arm64-graviton4-profile - -ssh-azure-linux-intel-gnr-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-intel-gnr-profile - -ssh-azure-linux-intel-emr-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-intel-emr-profile - -ssh-azure-linux-amd-zen5-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-amd-zen5-profile +ssh-create target *args="": + @"{{ dev_machine }}" create rscrypto "{{ target }}" {{ args }} -ssh-azure-linux-amd-zen4-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-amd-zen4-profile +ssh-start target: + @"{{ dev_machine }}" start rscrypto "{{ target }}" -ssh-azure-linux-arm64-cobalt-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-arm64-cobalt-profile +ssh-deallocate target: + @"{{ dev_machine }}" deallocate rscrypto "{{ target }}" -ssh-azure-linux-arm64-ampere-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-arm64-ampere-profile +ssh-kill target: + @"{{ dev_machine }}" kill rscrypto "{{ target }}" -ssh-azure-windows-amd-zen5-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-windows-amd-zen5-profile +ssh-status target="": + @if [ -n "{{ target }}" ]; then "{{ dev_machine }}" status rscrypto "{{ target }}"; else "{{ dev_machine }}" status rscrypto; fi -ssh-azure-windows-intel-gnr-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-windows-intel-gnr-profile +ssh-bootstrap target profile="": + @if [ -n "{{ profile }}" ]; then "{{ dev_machine }}" bootstrap rscrypto "{{ target }}" "{{ profile }}"; else "{{ dev_machine }}" bootstrap rscrypto "{{ target }}"; fi -ssh-azure-windows-arm64-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-windows-arm64-profile +ssh-just target *args="": + @"{{ dev_machine }}" just rscrypto "{{ target }}" {{ args }} -ssh-aws-test-rdma: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-test-rdma +ssh-collect-bench target run_id destination: + @"{{ dev_machine }}" collect-bench rscrypto "{{ target }}" "{{ run_id }}" "{{ destination }}" -ssh-azure-test-rdma: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-test-rdma +ssh-list: + @"{{ dev_machine }}" list # Builds build: - cargo build --workspace --all-targets --all-features + cargo build --locked --workspace --all-targets --all-features build-release: - cargo build --workspace --all-targets --all-features --release + cargo build --locked --workspace --all-targets --all-features --release # Checks check *args="": @@ -120,9 +65,9 @@ test-feature-matrix: @scripts/test/test-feature-matrix.sh test-native-api: - cargo test --no-default-features --features 'alloc,aead,ed25519,x25519,ecdsa,ml-kem' --test api_consistency - cargo test --features 'aead,signatures,key-exchange,getrandom' --test api_consistency - cargo test --features 'signatures,key-exchange,getrandom' --test getrandom_smoke + cargo test --locked --no-default-features --features 'alloc,aead,ed25519,x25519,ecdsa,ml-kem' --test api_consistency + cargo test --locked --features 'aead,signatures,key-exchange,getrandom' --test api_consistency + cargo test --locked --features 'signatures,key-exchange,getrandom' --test getrandom_smoke # Tests test *crates="": diff --git a/rust-toolchain.toml b/rust-toolchain.toml index f3cb4461..e81aa497 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,26 +1,4 @@ [toolchain] -channel = "nightly-2026-07-17" -profile = "default" -components = ["clippy", "rustfmt", "miri", "rust-src"] - -targets = [ - "aarch64-apple-darwin", - "aarch64-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "wasm32-unknown-unknown", - "wasm32-wasip1", - "x86_64-unknown-linux-musl", - "aarch64-pc-windows-msvc", - "aarch64-unknown-none", - "loongarch64-unknown-linux-gnu", - "powerpc64le-unknown-linux-gnu", - "riscv32imac-unknown-none-elf", - "riscv64gc-unknown-linux-gnu", - "riscv64gc-unknown-none-elf", - "s390x-unknown-linux-gnu", - "thumbv6m-none-eabi", - "x86_64-unknown-none", -] +channel = "1.97.1" +profile = "minimal" +components = ["clippy", "rustfmt", "rust-src", "rust-analyzer"] diff --git a/rustfmt.toml b/rustfmt.toml index 50910770..7fd03e83 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,22 +1,7 @@ edition = "2024" style_edition = "2024" -tab_spaces = 2 max_width = 120 hard_tabs = false - -reorder_imports = true -imports_granularity = "Crate" -group_imports = "StdExternalCrate" - -format_code_in_doc_comments = true -format_strings = true -normalize_comments = true -wrap_comments = true -comment_width = 100 - -use_field_init_shorthand = true -use_try_shorthand = true - -error_on_line_overflow = false -error_on_unformatted = false +tab_spaces = 2 +newline_style = "Unix" diff --git a/scripts/README.md b/scripts/README.md index 3bdd2d16..bc6beb14 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -73,6 +73,7 @@ claim in [`docs/secret-lifecycle.md`](../docs/secret-lifecycle.md). | `check/check-win.sh` | `scripts/check/check-all.sh` | | `check/check-linux.sh` | `scripts/check/check-all.sh` | | `check/check-ibm.sh` | `scripts/check/check-all.sh` | +| `check/lint-independent-workspaces.sh` | `scripts/check/check.sh --all` | | `check/zig-cc.sh` | `scripts/check/check-linux.sh`, `scripts/check/check-ibm.sh` | ## Bench Internals @@ -127,10 +128,10 @@ candidate selected after the metadata refresh, verifies the installed version, and refuses downgrades. OPAM uses exact packages from a repository pinned to a full Git commit and verifies package source hashes from that immutable metadata. CT formal reports bind the resulting BINSEC executable by SHA-256. Rustup -receives only the exact toolchain declared in `rust-toolchain.toml`; runner -images must provide rustup, which verifies component downloads against the -exact distribution manifest, because network bootstrap installers are -rejected. +receives only the exact stable or nightly contract declared in +`rust-toolchain.toml` and `.config/toolchains.toml`; runner images must provide +rustup, which verifies component downloads against the exact distribution +manifest, because network bootstrap installers are rejected. ## Results layout diff --git a/scripts/check/check-all.sh b/scripts/check/check-all.sh index 31ad9ff3..b13e6b4f 100755 --- a/scripts/check/check-all.sh +++ b/scripts/check/check-all.sh @@ -10,6 +10,8 @@ source "$SCRIPT_DIR/../lib/common.sh" # shellcheck source=../lib/targets.sh source "$SCRIPT_DIR/../lib/targets.sh" +NIGHTLY_TOOLCHAIN=$("$SCRIPT_DIR/../lib/toolchain.sh" --nightly) + DEFAULT_CONSTRAINED_CRATES=( "rscrypto" ) @@ -77,6 +79,16 @@ crate_supports_alloc() { [[ -f "$manifest" ]] && grep -q '^[[:space:]]*alloc[[:space:]]*=' "$manifest" } +cargo_for_target() { + local target=$1 + shift + if [[ "$target" == riscv32* ]]; then + RUSTUP_TOOLCHAIN="$NIGHTLY_TOOLCHAIN" cargo "$@" + else + cargo "$@" + fi +} + run_constrained_check() { local crate=$1 local target=$2 @@ -97,7 +109,7 @@ run_constrained_check() { args+=(--features "$feature_set") fi - if ! RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" cargo "${args[@]}" >>"$log_file" 2>&1; then + if ! RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" cargo_for_target "$target" "${args[@]}" >>"$log_file" 2>&1; then return 1 fi } @@ -106,7 +118,11 @@ run_constrained_target() { local target=$1 local log_dir=$2 - ensure_target "$target" + if [[ "$target" == riscv32* ]]; then + ensure_target "$target" "$NIGHTLY_TOOLCHAIN" + else + ensure_target "$target" + fi local target_dir="target/cross-check/$target" mkdir -p "$target_dir" @@ -147,7 +163,7 @@ run_constrained_target() { step "$target check (alloc)" for crate in "${alloc_crates[@]}"; do if ! RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo check --locked -p "$crate" --no-default-features --features alloc --target "$target" --lib \ + cargo_for_target "$target" check --locked -p "$crate" --no-default-features --features alloc --target "$target" --lib \ >>"$log_file" 2>&1; then fail show_error "$log_file" @@ -160,7 +176,7 @@ run_constrained_target() { step "$target build (no features)" for crate in "${CONSTRAINED_CRATES[@]}"; do if ! RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo build --locked -p "$crate" --no-default-features --target "$target" --lib --release \ + cargo_for_target "$target" build --locked -p "$crate" --no-default-features --target "$target" --lib --release \ >>"$log_file" 2>&1; then fail show_error "$log_file" @@ -173,7 +189,7 @@ run_constrained_target() { step "$target build (alloc)" for crate in "${alloc_crates[@]}"; do if ! RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo build --locked -p "$crate" --no-default-features --features alloc --target "$target" --lib --release \ + cargo_for_target "$target" build --locked -p "$crate" --no-default-features --features alloc --target "$target" --lib --release \ >>"$log_file" 2>&1; then fail show_error "$log_file" diff --git a/scripts/check/check-ibm.sh b/scripts/check/check-ibm.sh index c9ce4008..140f91ff 100755 --- a/scripts/check/check-ibm.sh +++ b/scripts/check/check-ibm.sh @@ -21,6 +21,7 @@ fi get_crate_flags "$@" export ZIG_CC="$SCRIPT_DIR/zig-cc.sh" +NIGHTLY_TOOLCHAIN=$("$SCRIPT_DIR/../lib/toolchain.sh" --nightly) LOG_DIR=$(mktemp -d) trap 'rm -rf "$LOG_DIR"' EXIT @@ -33,7 +34,7 @@ if [[ ${#IBM_TARGETS[@]} -eq 0 ]]; then fi for target in "${IBM_TARGETS[@]}"; do - ensure_target "$target" + ensure_target "$target" "$NIGHTLY_TOOLCHAIN" done for target in "${IBM_TARGETS[@]}"; do @@ -53,8 +54,8 @@ for i in "${!IBM_TARGETS[@]}"; do ( # shellcheck disable=SC2086 - if ! CC="$ZIG_CC" RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo clippy $CRATE_FLAGS --lib --all-features --locked --target "$target" -- -D warnings \ + if ! CC="$ZIG_CC" RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" RUSTUP_TOOLCHAIN="$NIGHTLY_TOOLCHAIN" \ + cargo clippy $CRATE_FLAGS --lib --all-features --locked --target "$target" \ >"$log_file" 2>&1; then exit 1 fi diff --git a/scripts/check/check-linux.sh b/scripts/check/check-linux.sh index f35420d3..54ee22d4 100755 --- a/scripts/check/check-linux.sh +++ b/scripts/check/check-linux.sh @@ -21,6 +21,7 @@ fi get_crate_flags "$@" export ZIG_CC="$SCRIPT_DIR/zig-cc.sh" +NIGHTLY_TOOLCHAIN=$("$SCRIPT_DIR/../lib/toolchain.sh" --nightly) LOG_DIR=$(mktemp -d) trap 'rm -rf "$LOG_DIR"' EXIT @@ -33,7 +34,11 @@ if [[ ${#LINUX_TARGETS[@]} -eq 0 ]]; then fi for target in "${LINUX_TARGETS[@]}"; do - ensure_target "$target" + if [[ "$target" == riscv64* ]]; then + ensure_target "$target" "$NIGHTLY_TOOLCHAIN" + else + ensure_target "$target" + fi done for target in "${LINUX_TARGETS[@]}"; do @@ -52,9 +57,13 @@ for i in "${!LINUX_TARGETS[@]}"; do targets[i]="$target" ( + toolchain_env=() + if [[ "$target" == riscv64* ]]; then + toolchain_env=(env "RUSTUP_TOOLCHAIN=$NIGHTLY_TOOLCHAIN") + fi # shellcheck disable=SC2086 if ! CC="$ZIG_CC" RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo clippy $CRATE_FLAGS --lib --all-features --locked --target "$target" -- -D warnings \ + "${toolchain_env[@]}" cargo clippy $CRATE_FLAGS --lib --all-features --locked --target "$target" \ >"$log_file" 2>&1; then exit 1 fi diff --git a/scripts/check/check-win.sh b/scripts/check/check-win.sh index decebde9..33164498 100755 --- a/scripts/check/check-win.sh +++ b/scripts/check/check-win.sh @@ -85,7 +85,7 @@ for i in "${!WIN_TARGETS[@]}"; do # shellcheck disable=SC2086 if ! XWIN_CACHE_DIR="$target_cache_dir" \ CARGO_TARGET_DIR="$target_dir" \ - cargo xwin clippy $CRATE_FLAGS --lib --all-features --locked --target "$target" -- -D warnings \ + cargo xwin clippy $CRATE_FLAGS --lib --all-features --locked --target "$target" \ >"$log_file" 2>&1; then exit 1 fi diff --git a/scripts/check/check.sh b/scripts/check/check.sh index c9995b5d..7dd75b74 100755 --- a/scripts/check/check.sh +++ b/scripts/check/check.sh @@ -145,13 +145,23 @@ fi # Clippy step "Linting" # shellcheck disable=SC2086 -if ! cargo clippy $CRATE_FLAGS --all-targets --all-features --locked -- -D warnings >"$LOG_DIR/clippy.log" 2>&1; then +if ! cargo clippy $CRATE_FLAGS --all-targets --all-features --locked >"$LOG_DIR/clippy.log" 2>&1; then fail show_error "$LOG_DIR/clippy.log" exit 1 fi ok +if [[ "$FULL_WORKSPACE" == true ]]; then + step "Linting independent workspaces" + if ! "$SCRIPT_DIR/lint-independent-workspaces.sh" >"$LOG_DIR/independent-lints.log" 2>&1; then + fail + show_error "$LOG_DIR/independent-lints.log" + exit 1 + fi + ok +fi + # Audit/Deny (workspace only). CI owns this in the dedicated supply-chain lane. if [[ "$FULL_WORKSPACE" == true && "${RSCRYPTO_SKIP_CHECK_SUPPLY_CHAIN:-}" != "1" ]]; then step "Auditing deps" diff --git a/scripts/check/lint-independent-workspaces.sh b/scripts/check/lint-independent-workspaces.sh new file mode 100755 index 00000000..6dac5565 --- /dev/null +++ b/scripts/check/lint-independent-workspaces.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd -P)" +POLICY_MANIFEST="$REPO_ROOT/Cargo.toml" +TARGET_DIR="${RSCRYPTO_INDEPENDENT_LINT_TARGET_DIR:-$REPO_ROOT/target/independent-lints}" +MESSAGE_FORMAT="${RSCRYPTO_LINT_MESSAGE_FORMAT:-human}" +LINT_CAP="${RSCRYPTO_LINT_CAP:-}" + +case "$MESSAGE_FORMAT" in + human | json) ;; + *) echo "RSCRYPTO_LINT_MESSAGE_FORMAT must be human or json" >&2; exit 2 ;; +esac +case "$LINT_CAP" in + "" | warn) ;; + *) echo "RSCRYPTO_LINT_CAP must be empty or warn" >&2; exit 2 ;; +esac + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to resolve independent Cargo workspaces" >&2 + exit 1 +fi + +manifests=() +while IFS= read -r package_manifest; do + if ! metadata=$(cargo metadata \ + --locked \ + --no-deps \ + --format-version 1 \ + --manifest-path "$package_manifest"); then + echo "Failed to resolve Cargo workspace for $package_manifest" >&2 + exit 1 + fi + workspace_root=$(printf '%s\n' "$metadata" | jq -er '.workspace_root') + workspace_root=$(cd "$workspace_root" && pwd -P) + if [[ "$workspace_root" == "$REPO_ROOT" ]]; then + continue + fi + + workspace_manifest="$workspace_root/Cargo.toml" + already_listed=false + for manifest in "${manifests[@]:+${manifests[@]}}"; do + if [[ "$manifest" == "$workspace_manifest" ]]; then + already_listed=true + break + fi + done + if [[ "$already_listed" == false ]]; then + manifests+=("$workspace_manifest") + fi +done < <(find "$REPO_ROOT" -type d -name target -prune -o -type f -name Cargo.toml -print | sort) + +if [[ ${#manifests[@]} -eq 0 ]]; then + echo "No independent Cargo workspaces found through Cargo metadata" + exit 0 +fi + +lint_names() { + local namespace=$1 + awk -v section="[lints.$namespace]" ' + /^\[/ { + active = ($0 == section) + next + } + active { + line = $0 + sub(/^[[:space:]]*/, "", line) + if (line ~ /^[a-z0-9_]+[[:space:]]*=/ && line ~ /"deny"/) { + sub(/[[:space:]]*=.*/, "", line) + print line + } + } + ' "$POLICY_MANIFEST" +} + +lint_flags=() +while IFS= read -r lint; do + [[ -n "$lint" ]] && lint_flags+=("-D$lint") +done < <(lint_names rust) +while IFS= read -r lint; do + [[ -n "$lint" ]] && lint_flags+=("-Dclippy::$lint") +done < <(lint_names clippy) + +if [[ ${#lint_flags[@]} -eq 0 ]]; then + echo "No deny-level Rust or Clippy policy found in $POLICY_MANIFEST" >&2 + exit 1 +fi + +check_cfg_flags=( + '--check-cfg=cfg(miri)' + '--check-cfg=cfg(fuzzing)' + '--check-cfg=cfg(rscrypto_internal_fuzzing)' + '--check-cfg=cfg(target_feature,values("movdiri","movdir64b","serialize"))' +) + +failed=0 +for manifest in "${manifests[@]}"; do + relative_manifest=${manifest#"$REPO_ROOT/"} + if [[ "$MESSAGE_FORMAT" == json ]]; then + echo "Linting independent workspace: $relative_manifest" >&2 + else + echo "Linting independent workspace: $relative_manifest" + fi + + cargo_args=( + clippy + --locked + --manifest-path "$manifest" + --workspace + --all-targets + --all-features + --no-deps + ) + if [[ "$MESSAGE_FORMAT" == json ]]; then + cargo_args+=(--message-format=json) + fi + + compiler_flags=("${lint_flags[@]}" "${check_cfg_flags[@]}") + if [[ -n "$LINT_CAP" ]]; then + compiler_flags+=(--cap-lints "$LINT_CAP") + fi + + if ! CARGO_TARGET_DIR="$TARGET_DIR" cargo "${cargo_args[@]}" -- "${compiler_flags[@]}"; then + failed=1 + fi +done + +exit "$failed" diff --git a/scripts/check/rsa-asm-provenance.sh b/scripts/check/rsa-asm-provenance.sh index 65bd3051..a7941399 100755 --- a/scripts/check/rsa-asm-provenance.sh +++ b/scripts/check/rsa-asm-provenance.sh @@ -156,9 +156,9 @@ verify_rsa_wrapper_fingerprints() { [[ "$actual" == "$expected" ]] \ || die "$wrapper changed without a provenance review" done <<'EOF' -src/auth/rsa_aarch64_asm.rs 65c059365bb76d948d2f09873870ba7e146af15e52e10560bf632d39bdd33834 -src/auth/rsa_aarch64_linux_asm.rs 2b311eb4f253ff458492c76f52f35c0a4d5ae8705774e568ae853e1f6ea1a049 -src/auth/rsa_x86_64_asm.rs 98bc241435de70963c23f0e98431024585f95eed268b773a42d3bd2983b8038e +src/auth/rsa_aarch64_asm.rs dbcd55bc88daa1a9ab1812e964c84bfd4586ae9f6fa9b07dcd28985df6576507 +src/auth/rsa_aarch64_linux_asm.rs 69531bafa963d6d817c6a74f4a0fcaf1d98e713611874f4597768b1a6e0a0e7a +src/auth/rsa_x86_64_asm.rs 5552b8ffa76f1ba43f6aaea1b91a28fb04a02fa167408d925bf5ccec6daefe18 EOF } diff --git a/scripts/ci/changed-test-planning-test.sh b/scripts/ci/changed-test-planning-test.sh index 470aa82f..855e7902 100755 --- a/scripts/ci/changed-test-planning-test.sh +++ b/scripts/ci/changed-test-planning-test.sh @@ -226,7 +226,7 @@ run_test_consumer() { assert_eq "$expected" "$actual" "test commands for $name" } -workspace_test='cargo test --workspace --all-features --lib --tests' +workspace_test='cargo test --locked --workspace --all-features --lib --tests' run_test_consumer planner-failure '' 9 "$workspace_test" run_test_consumer empty-output '' 0 "$workspace_test" run_test_consumer malformed-json '{' 0 "$workspace_test" @@ -238,12 +238,13 @@ run_test_consumer malformed-crates "$empty_crate_selection" 0 "$workspace_test" run_test_consumer malformed-surface "$malformed_surface" 0 "$workspace_test" run_test_consumer valid-empty "$EMPTY_PLAN" 0 '' run_test_consumer valid-workspace "$WORKSPACE_PLAN" 0 "$workspace_test" -run_test_consumer valid-crates "$CRATES_PLAN" 0 $'cargo test -p crate-a --all-features --lib --tests\ncargo test -p crate-b --all-features --lib --tests' +run_test_consumer valid-crates "$CRATES_PLAN" 0 $'cargo test --locked -p crate-a --all-features --lib --tests\ncargo test --locked -p crate-b --all-features --lib --tests' run_test_consumer valid-legacy-workspace "$LEGACY_WORKSPACE_PLAN" 0 "$workspace_test" check_fixture="$TMP_ROOT/check-repository" mkdir -p "$check_fixture/scripts/check" "$check_fixture/scripts/lib" "$check_fixture/scripts/ct" "$check_fixture/scripts/test" cp "$REPO_ROOT/scripts/check/check.sh" "$check_fixture/scripts/check/check.sh" +cp "$REPO_ROOT/scripts/check/lint-independent-workspaces.sh" "$check_fixture/scripts/check/lint-independent-workspaces.sh" cp "$REPO_ROOT/scripts/lib/common.sh" "$REPO_ROOT/scripts/lib/rail-plan.sh" "$check_fixture/scripts/lib/" for helper in "$check_fixture/scripts/check/asm-ledger.sh" \ "$check_fixture/scripts/check/check-feature-matrix.sh" \ @@ -289,9 +290,11 @@ run_check_consumer valid-workspace "$WORKSPACE_PLAN" 0 '--workspace' run_check_consumer valid-crates "$CRATES_PLAN" 0 '-p crate-a -p crate-b' check_all_fixture="$TMP_ROOT/check-all-repository" -mkdir -p "$check_all_fixture/scripts/check" "$check_all_fixture/scripts/lib" +mkdir -p "$check_all_fixture/.config" "$check_all_fixture/scripts/check" "$check_all_fixture/scripts/lib" cp "$REPO_ROOT/scripts/check/check-all.sh" "$check_all_fixture/scripts/check/check-all.sh" -cp "$REPO_ROOT/scripts/lib/common.sh" "$REPO_ROOT/scripts/lib/rail-plan.sh" "$check_all_fixture/scripts/lib/" +cp "$REPO_ROOT/scripts/lib/common.sh" "$REPO_ROOT/scripts/lib/rail-plan.sh" \ + "$REPO_ROOT/scripts/lib/toolchain.sh" "$check_all_fixture/scripts/lib/" +cp "$REPO_ROOT/.config/toolchains.toml" "$check_all_fixture/.config/toolchains.toml" cat >"$check_all_fixture/scripts/lib/targets.sh" <<'EOF' WIN_TARGETS=() LINUX_TARGETS=() diff --git a/scripts/ci/check-ci-ownership-test.sh b/scripts/ci/check-ci-ownership-test.sh index 781883e6..c0298ad2 100755 --- a/scripts/ci/check-ci-ownership-test.sh +++ b/scripts/ci/check-ci-ownership-test.sh @@ -128,7 +128,7 @@ expect_failure "$invalid_tool_digest" "direct tool digest is malformed" mutable_tool_url="$TMP_ROOT/mutable-tool-url" make_fixture "$mutable_tool_url" -sed -i.bak 's#/download/v46\.0\.1/#/download/Latest/#' \ +sed -i.bak 's#/download/v47\.0\.3/#/download/Latest/#' \ "$mutable_tool_url/.config/ci-tool-archives.tsv" rm -f "$mutable_tool_url/.config/ci-tool-archives.tsv.bak" expect_failure "$mutable_tool_url" "direct tool URL resolves a mutable release" diff --git a/scripts/ci/check-ci-ownership.sh b/scripts/ci/check-ci-ownership.sh index d83c9e17..e9aa93a2 100755 --- a/scripts/ci/check-ci-ownership.sh +++ b/scripts/ci/check-ci-ownership.sh @@ -686,7 +686,7 @@ group_musl=$(jq '[.groups.linux[] | select(contains("musl"))] | length' "$MANIFE [[ "$group_musl" -eq 2 ]] || fail "the target manifest must retain both MUSL triples" # shellcheck disable=SC2016 # `$target` is an intentional literal in the workflow contract regex. -[[ $(count_matches 'cargo (check|clippy|build) --target "\$target"' "$CROSS_SCRIPT") -ge 3 ]] \ +[[ $(count_matches 'cargo (check|clippy|build) --locked --target "\$target"' "$CROSS_SCRIPT") -ge 3 ]] \ || fail "MUSL evidence must pass the target triple explicitly to Cargo" echo "CI ownership contract passed" diff --git a/scripts/ci/check-locked-cargo-test.sh b/scripts/ci/check-locked-cargo-test.sh new file mode 100755 index 00000000..8ee6480e --- /dev/null +++ b/scripts/ci/check-locked-cargo-test.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHECKER="$SCRIPT_DIR/check-locked-cargo.sh" +TMP_ROOT=$(mktemp -d) +trap 'rm -rf "$TMP_ROOT"' EXIT + +fail() { + echo "locked Cargo inventory test failure: $*" >&2 + exit 1 +} + +mkdir -p "$TMP_ROOT/scripts/ci" "$TMP_ROOT/.zed" +printf '%s\n' 'build:' ' cargo build --locked --workspace' >"$TMP_ROOT/justfile" +"$CHECKER" --root "$TMP_ROOT" || fail "locked command was rejected" + +printf '%s\n' '#!/usr/bin/env bash' 'cargo test --workspace' >"$TMP_ROOT/scripts/ci/example.sh" +if "$CHECKER" --root "$TMP_ROOT" >/dev/null 2>&1; then + fail "unlocked command was accepted" +fi + +printf '%s\n' '#!/usr/bin/env bash' 'cargo test \' ' --locked \' ' --workspace' >"$TMP_ROOT/scripts/ci/example.sh" +"$CHECKER" --root "$TMP_ROOT" || fail "multiline locked command was rejected" + +printf '%s\n' '#!/usr/bin/env bash' '# cargo check --workspace' >"$TMP_ROOT/scripts/ci/example.sh" +"$CHECKER" --root "$TMP_ROOT" || fail "comment was treated as a command" + +printf '%s\n' '[{"label":"check","command":"cargo","args":["check"]}]' >"$TMP_ROOT/.zed/tasks.json" +if "$CHECKER" --root "$TMP_ROOT" >/dev/null 2>&1; then + fail "unlocked Zed Cargo task was accepted" +fi + +printf '%s\n' '[{"label":"check","command":"just","args":["check"]}]' >"$TMP_ROOT/.zed/tasks.json" +"$CHECKER" --root "$TMP_ROOT" || fail "repository-front-door Zed task was rejected" + +echo "Locked Cargo inventory regression tests passed" diff --git a/scripts/ci/check-locked-cargo.sh b/scripts/ci/check-locked-cargo.sh new file mode 100755 index 00000000..ddd1db65 --- /dev/null +++ b/scripts/ci/check-locked-cargo.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="" +if [[ ${1:-} == --root ]]; then + ROOT=${2:?missing path after --root} + shift 2 +fi +if [[ $# -ne 0 ]]; then + echo "usage: check-locked-cargo.sh [--root PATH]" >&2 + exit 2 +fi + +if [[ -z "$ROOT" ]]; then + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +fi + +status=0 +check_statement() { + local file=$1 + local line_number=$2 + local statement=$3 + local trimmed=${statement#"${statement%%[![:space:]]*}"} + + case "$trimmed" in + echo\ * | printf\ * | step\ *) return ;; + esac + + if [[ "$statement" =~ (^|[[:space:]])cargo[[:space:]]+(build|check|clippy|test|bench|rustc|run|doc|miri|nextest)($|[[:space:]]) ]] \ + && [[ "$statement" != *"--locked"* ]]; then + echo "$file:$line_number: routine Cargo command must use --locked" >&2 + echo " $statement" >&2 + status=1 + fi + + if [[ "$statement" =~ (^|[[:space:]])cargo[[:space:]]+llvm-cov[[:space:]]+(nextest|test)($|[[:space:]]) ]] \ + && [[ "$statement" != *"--locked"* ]]; then + echo "$file:$line_number: cargo-llvm-cov execution must use --locked" >&2 + echo " $statement" >&2 + status=1 + fi +} + +while IFS= read -r file; do + statement="" + statement_line=0 + line_number=0 + while IFS= read -r line || [[ -n "$line" ]]; do + line_number=$((line_number + 1)) + if [[ -z "$statement" ]]; then + [[ "$line" =~ ^[[:space:]]*# ]] && continue + statement_line=$line_number + fi + statement+=" ${line%\\}" + if [[ "$line" == *\\ ]]; then + continue + fi + check_statement "${file#"$ROOT"/}" "$statement_line" "$statement" + statement="" + done <"$file" + if [[ -n "$statement" ]]; then + check_statement "${file#"$ROOT"/}" "$statement_line" "$statement" + fi +done < <( + { + [[ -f "$ROOT/justfile" ]] && printf '%s\n' "$ROOT/justfile" + find "$ROOT/scripts" -type f -name '*.sh' \ + ! -name '*-test.sh' ! -name '*-scheduler-test.sh' 2>/dev/null || true + find "$ROOT/.github" -type f \( -name '*.yaml' -o -name '*.yml' \) 2>/dev/null || true + } | LC_ALL=C sort +) + +if [[ -f "$ROOT/.zed/tasks.json" ]]; then + while IFS= read -r task; do + command=$(jq -r '.command' <<<"$task") + args=$(jq -r '(.args // []) | join(" ")' <<<"$task") + check_statement ".zed/tasks.json" 1 "$command $args" + done < <(jq -c '.[] | select((.command // "") == "cargo" or ((.command // "") | startswith("cargo ")))' \ + "$ROOT/.zed/tasks.json") +fi + +exit "$status" diff --git a/scripts/ci/check-worktree-test.sh b/scripts/ci/check-worktree-test.sh index e05ff043..f0ff3788 100755 --- a/scripts/ci/check-worktree-test.sh +++ b/scripts/ci/check-worktree-test.sh @@ -17,6 +17,7 @@ fake_home="$TMP_ROOT/home" command_log="$TMP_ROOT/commands.log" preflight_marker="$TMP_ROOT/locked-metadata-preflight" mkdir -p \ + "$fixture/.config" \ "$fixture/scripts/check" \ "$fixture/scripts/ct" \ "$fixture/scripts/lib" \ @@ -31,12 +32,15 @@ cp \ "$REPO_ROOT/scripts/check/check-feature-matrix.sh" \ "$REPO_ROOT/scripts/check/check-ibm.sh" \ "$REPO_ROOT/scripts/check/check-linux.sh" \ + "$REPO_ROOT/scripts/check/lint-independent-workspaces.sh" \ "$REPO_ROOT/scripts/check/check-win.sh" \ "$REPO_ROOT/scripts/check/check.sh" \ "$REPO_ROOT/scripts/check/zig-cc.sh" \ "$fixture/scripts/check/" cp "$REPO_ROOT/scripts/lib/common.sh" "$REPO_ROOT/scripts/lib/rail-plan.sh" \ - "$REPO_ROOT/scripts/lib/feature-profiles.sh" "$fixture/scripts/lib/" + "$REPO_ROOT/scripts/lib/feature-profiles.sh" "$REPO_ROOT/scripts/lib/toolchain.sh" \ + "$fixture/scripts/lib/" +cp "$REPO_ROOT/.config/toolchains.toml" "$fixture/.config/toolchains.toml" cp "$REPO_ROOT/scripts/test/test-feature-matrix.sh" "$fixture/scripts/test/test-feature-matrix.sh" cat >"$fixture/scripts/lib/targets.sh" <<'EOF' @@ -104,6 +108,7 @@ case "${1:-}" in metadata) require_locked "$@" : >"$MOCK_PREFLIGHT_MARKER" + printf '{"workspace_root":"%s"}\n' "$MOCK_REPO_ROOT" ;; rail) if [[ "${2:-}" == "plan" ]]; then diff --git a/scripts/ci/ci-check.sh b/scripts/ci/ci-check.sh index 700023ba..0af74976 100755 --- a/scripts/ci/ci-check.sh +++ b/scripts/ci/ci-check.sh @@ -14,6 +14,9 @@ echo "" echo "🔗 Checking workflow ownership and action pins..." "$SCRIPT_DIR/check-ci-ownership.sh" "$SCRIPT_DIR/check-ci-ownership-test.sh" +"$SCRIPT_DIR/check-locked-cargo.sh" +"$SCRIPT_DIR/check-locked-cargo-test.sh" +"$SCRIPT_DIR/../lib/target-matrix.sh" --validate "$SCRIPT_DIR/check-action-pins.sh" export RSCRYPTO_SKIP_CHECK_SUPPLY_CHAIN=1 @@ -21,7 +24,7 @@ export RSCRYPTO_SKIP_CHECK_SUPPLY_CHAIN=1 echo "" echo "🔨 Building all targets..." -cargo build --workspace --all-targets --all-features +cargo build --locked --workspace --all-targets --all-features echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/scripts/ci/cross-targets.sh b/scripts/ci/cross-targets.sh index 044312e7..01784b29 100755 --- a/scripts/ci/cross-targets.sh +++ b/scripts/ci/cross-targets.sh @@ -36,11 +36,11 @@ for target in "${MUSL_TARGETS[@]}"; do mkdir -p "$target_dir" RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo check --target "$target" --no-default-features --lib + cargo check --locked --target "$target" --no-default-features --lib RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo clippy --target "$target" --lib --all-features -- -D warnings + cargo clippy --locked --target "$target" --lib --all-features RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo build --target "$target" --no-default-features --features alloc --lib --release + cargo build --locked --target "$target" --no-default-features --features alloc --lib --release done for target in "${NOSTD_TARGETS[@]}" "${WASM_TARGETS[@]}"; do diff --git a/scripts/ci/install-tools.sh b/scripts/ci/install-tools.sh index a5dcea6a..e354cd52 100755 --- a/scripts/ci/install-tools.sh +++ b/scripts/ci/install-tools.sh @@ -6,13 +6,13 @@ set -euo pipefail MODE=${1:-standard} -CARGO_NEXTEST_VERSION=0.9.140 +CARGO_NEXTEST_VERSION=0.9.143 CARGO_DENY_VERSION=0.20.2 CARGO_AUDIT_VERSION=0.22.2 -CARGO_RAIL_VERSION=0.20.0 +CARGO_RAIL_VERSION=0.21.0 CARGO_SEMVER_CHECKS_VERSION=0.50.0 -JUST_VERSION=1.57.0 -ZIZMOR_VERSION=1.26.1 +JUST_VERSION=1.58.0 +ZIZMOR_VERSION=1.29.0 CARGO_CRITERION_VERSION=1.1.0 CRITCMP_VERSION=0.1.8 CARGO_FUZZ_VERSION=0.13.2 diff --git a/scripts/ci/mlkem-aarch64-gate.sh b/scripts/ci/mlkem-aarch64-gate.sh index babf284d..20f6ed36 100755 --- a/scripts/ci/mlkem-aarch64-gate.sh +++ b/scripts/ci/mlkem-aarch64-gate.sh @@ -60,22 +60,22 @@ else fi run_step "owned aarch64 NEON NTT scalar oracle" \ - cargo test --lib --features ml-kem ntt_neon -- --nocapture + cargo test --locked --lib --features ml-kem ntt_neon -- --nocapture run_step "owned aarch64 basemul scalar oracle" \ - cargo test --lib --features ml-kem basemul_accumulate -- --nocapture + cargo test --locked --lib --features ml-kem basemul_accumulate -- --nocapture run_step "ML-KEM ACVP FIPS 203 vectors" \ - cargo test --test mlkem_acvp --features ml-kem -- --nocapture + cargo test --locked --test mlkem_acvp --features ml-kem -- --nocapture run_step "ML-KEM operation tests" \ - cargo test --test mlkem_ops --features ml-kem -- --nocapture + cargo test --locked --test mlkem_ops --features ml-kem -- --nocapture run_step "ML-KEM FIPS/property tests" \ - cargo test --test mlkem_properties --features ml-kem -- --nocapture + cargo test --locked --test mlkem_properties --features ml-kem -- --nocapture run_step "ML-KEM fuzz corpus replay" \ - cargo test --manifest-path fuzz/Cargo.toml --test corpus_replay replay_auth_mlkem -- --nocapture + cargo test --locked --manifest-path fuzz/Cargo.toml --test corpus_replay replay_auth_mlkem -- --nocapture if [[ "$(bool_value "${RSCRYPTO_MLKEM_AARCH64_GATE_SKIP_BENCH:-false}")" == "true" ]]; then echo "warning: ML-KEM benchmark gate skipped by RSCRYPTO_MLKEM_AARCH64_GATE_SKIP_BENCH" | tee -a "$LOG_PATH" diff --git a/scripts/ci/native-check.sh b/scripts/ci/native-check.sh index 13742b84..dba8f39a 100755 --- a/scripts/ci/native-check.sh +++ b/scripts/ci/native-check.sh @@ -23,7 +23,7 @@ echo "Native validation: $(rustc -vV | sed -n 's/^host: //p')" echo "" echo "Checking no-default-features library boundary..." -cargo check --workspace --lib --no-default-features +cargo check --locked --workspace --lib --no-default-features TARGET_ARGS=(--lib) if [[ "$ALL_TARGETS" == true ]]; then @@ -32,16 +32,16 @@ fi echo "" echo "Checking all-feature native targets..." -cargo check --workspace "${TARGET_ARGS[@]}" --all-features +cargo check --locked --workspace "${TARGET_ARGS[@]}" --all-features echo "" echo "Linting all-feature native targets..." -cargo clippy --workspace "${TARGET_ARGS[@]}" --all-features -- -D warnings +cargo clippy --locked --workspace "${TARGET_ARGS[@]}" --all-features if [[ "$ALL_TARGETS" == true ]]; then echo "" echo "Building all-feature native targets..." - cargo build --workspace --all-targets --all-features + cargo build --locked --workspace --all-targets --all-features fi echo "Native validation passed" diff --git a/scripts/ci/nostd-wasm-suite.sh b/scripts/ci/nostd-wasm-suite.sh index 0125984f..070bc890 100755 --- a/scripts/ci/nostd-wasm-suite.sh +++ b/scripts/ci/nostd-wasm-suite.sh @@ -57,9 +57,9 @@ run_wasm_runtime_vectors() { export CARGO_TARGET_WASM32_WASIP1_RUNNER="wasmtime" local manifest="tools/wasm-runtime-vectors/Cargo.toml" - cargo run --manifest-path "$manifest" --target "$TARGET" + cargo run --locked --manifest-path "$manifest" --target "$TARGET" RUSTFLAGS="-C target-feature=+simd128" \ - cargo run --manifest-path "$manifest" --target "$TARGET" + cargo run --locked --manifest-path "$manifest" --target "$TARGET" } echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @@ -67,11 +67,11 @@ echo "Cross-compile sweep: $TARGET ($DEPTH)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Always: bare no-default-features. -cargo check --target "$TARGET" --no-default-features --lib -cargo build --target "$TARGET" --no-default-features --lib --release +cargo check --locked --target "$TARGET" --no-default-features --lib +cargo build --locked --target "$TARGET" --no-default-features --lib --release # Always: alloc. -cargo check --target "$TARGET" --no-default-features --features alloc --lib +cargo check --locked --target "$TARGET" --no-default-features --features alloc --lib if [[ "$DEPTH" == "deep" ]]; then # Union of the historical check-all facade matrix and the dedicated weekly @@ -110,22 +110,22 @@ if [[ "$DEPTH" == "deep" ]]; then ) for feature_set in "${FEATURE_SETS[@]}"; do - cargo check --target "$TARGET" --no-default-features --features "$feature_set" --lib + cargo check --locked --target "$TARGET" --no-default-features --features "$feature_set" --lib done # Full no_std release build. - cargo build --target "$TARGET" --no-default-features --features "alloc,checksums,hashes,auth,aead" --lib --release + cargo build --locked --target "$TARGET" --no-default-features --features "alloc,checksums,hashes,auth,aead" --lib --release fi # Target-specific smoke (shallow gets a token extra so each target has >0 feature coverage). if [[ "$DEPTH" == "shallow" ]]; then case "$TARGET" in thumbv6m-none-eabi) - cargo check --target "$TARGET" --no-default-features --features checksums --lib + cargo check --locked --target "$TARGET" --no-default-features --features checksums --lib ;; wasm32-unknown-unknown) - cargo check --target "$TARGET" --no-default-features --features hashes --lib - RUSTFLAGS="-C target-feature=+simd128" cargo check --target "$TARGET" --no-default-features --features hashes --lib + cargo check --locked --target "$TARGET" --no-default-features --features hashes --lib + RUSTFLAGS="-C target-feature=+simd128" cargo check --locked --target "$TARGET" --no-default-features --features hashes --lib ;; esac fi diff --git a/scripts/ci/release-identity-test.sh b/scripts/ci/release-identity-test.sh index 2a32782d..aa43fcab 100755 --- a/scripts/ci/release-identity-test.sh +++ b/scripts/ci/release-identity-test.sh @@ -96,7 +96,7 @@ jq -e --arg commit "$commit" ' and (.release.tag_object | test("^[0-9a-f]{40}$")) and .release.git_commit == $commit and (.release.git_tree | test("^[0-9a-f]{40}$")) - and .toolchain.channel == "nightly-2026-07-17" + and .toolchain.channel == "1.97.1" and (.toolchain.manifest.sha256 | test("^[0-9a-f]{64}$")) and .evidence.git_commit == $commit and .evidence.mode == "exact_commit" diff --git a/scripts/ci/run-bench.sh b/scripts/ci/run-bench.sh index a693ccb3..cec94cf6 100755 --- a/scripts/ci/run-bench.sh +++ b/scripts/ci/run-bench.sh @@ -916,7 +916,7 @@ run_bench_cmd() { bench_features="$(bench_features_for_target "$bench")" cargo_bench="$(bench_binary_for_target "$bench")" - cmd=(cargo bench --profile bench --features "$bench_features" --bench "$cargo_bench") + cmd=(cargo bench --locked --profile bench --features "$bench_features" --bench "$cargo_bench") if [[ -n "$filter" || "${#CRITERION_ARGS[@]}" -gt 0 ]]; then cmd+=(--) if [[ -n "$filter" ]]; then @@ -997,7 +997,7 @@ fi GENERIC_FEATURES="$(bench_features_for_invocation "$BENCHES_INPUT")" echo "Using features: $GENERIC_FEATURES" | tee -a "$LOG_PATH" -cmd=(cargo bench --profile bench --features "$GENERIC_FEATURES") +cmd=(cargo bench --locked --profile bench --features "$GENERIC_FEATURES") if [[ "${#BENCH_FLAGS[@]}" -gt 0 ]]; then cmd+=("${BENCH_FLAGS[@]}") fi diff --git a/scripts/ci/run-rust-job-test.sh b/scripts/ci/run-rust-job-test.sh index 6b0f0769..4839918b 100755 --- a/scripts/ci/run-rust-job-test.sh +++ b/scripts/ci/run-rust-job-test.sh @@ -292,7 +292,7 @@ grep -Fq \ "$CAPTURE/amx-cargo.args" \ || fail "AMX integration test existence was not checked under the required permission contract" grep -Fq \ - 'RUSTFLAGS=-A unstable-features -C target-feature=+amx-tile,+amx-bf16,+amx-int8' \ + 'RUSTFLAGS=-C target-feature=+amx-tile,+amx-bf16,+amx-int8' \ "$CAPTURE/amx-cargo.args" \ || fail "AMX no_std test existence was not checked with forced AMX target features" diff --git a/scripts/ci/run-rust-job.sh b/scripts/ci/run-rust-job.sh index 9b3eca8e..a5a77cb4 100755 --- a/scripts/ci/run-rust-job.sh +++ b/scripts/ci/run-rust-job.sh @@ -70,6 +70,11 @@ run_quality() { just ci-check } +run_msrv() { + cargo check --locked --workspace --lib --no-default-features + cargo check --locked --workspace --lib --all-features +} + run_cargo_graph() { mkdir -p target/cargo-rail cargo rail config validate --strict @@ -97,16 +102,16 @@ run_native() { case "$target" in x86_64-pc-windows-msvc) echo "Windows x86_64 lane: compile smoke plus BLAKE3 ASM runtime vectors" - cargo clippy --workspace --lib --all-features -- -D warnings - cargo test --workspace --all-features --no-run - cargo test --workspace --features blake3 \ + cargo clippy --locked --workspace --lib --all-features + cargo test --locked --workspace --all-features --no-run + cargo test --locked --workspace --features blake3 \ --test blake3_official_vectors \ --test blake3_differential ;; aarch64-pc-windows-msvc) echo "Windows AArch64 lane: compile-only smoke" - cargo clippy --workspace --lib --all-features -- -D warnings - cargo test --workspace --all-features --no-run + cargo clippy --locked --workspace --lib --all-features + cargo test --locked --workspace --all-features --no-run ;; x86_64-unknown-linux-gnu | aarch64-unknown-linux-gnu) bash scripts/ci/native-check.sh --all-targets @@ -161,7 +166,7 @@ run_platform_amx() { # NIGHTLY: Rust target-feature names for AMX remain unstable. This lane # deliberately forces them so the no_std permission gate is executable. - local amx_rustflags="-A unstable-features -C target-feature=+amx-tile,+amx-bf16,+amx-int8" + local amx_rustflags="-C target-feature=+amx-tile,+amx-bf16,+amx-int8" RUSTFLAGS="$amx_rustflags" \ assert_single_libtest \ platform::detect::tests::no_std_linux_x86_64_masks_compile_time_amx_without_a_permission_probe \ @@ -460,6 +465,7 @@ main() { require_nonempty operation "$operation" case "$operation" in quality) run_quality ;; + msrv) run_msrv ;; cargo-graph) run_cargo_graph ;; feature-contracts) run_feature_contracts ;; native) run_native ;; diff --git a/scripts/ci/tool-integrity-test.sh b/scripts/ci/tool-integrity-test.sh index 5daedf29..126c0dac 100755 --- a/scripts/ci/tool-integrity-test.sh +++ b/scripts/ci/tool-integrity-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" @@ -90,11 +91,11 @@ while [[ $# -gt 0 ]]; do esac done [[ -n "$destination" ]] -root="$destination/wasmtime-v46.0.1-x86_64-linux" +root="$destination/wasmtime-v47.0.3-x86_64-linux" mkdir -p "$root" cat >"$root/wasmtime" <<'EOF' #!/usr/bin/env bash -printf 'wasmtime 46.0.1 (mock)\n' +printf 'wasmtime 47.0.3 (mock)\n' printf 'wasmtime executed\n' >>"$MOCK_EXEC_LOG" EOF chmod +x "$root/wasmtime" @@ -537,13 +538,13 @@ if HOME="$ct_home" \ fi for contract in \ - 'cargo-nextest =0.9.140' \ + 'cargo-nextest =0.9.143' \ 'cargo-deny =0.20.2' \ 'cargo-audit =0.22.2' \ - 'cargo-rail =0.20.0' \ + 'cargo-rail =0.21.0' \ 'cargo-semver-checks =0.50.0' \ - 'just =1.57.0' \ - 'zizmor =1.26.1' \ + 'just =1.58.0' \ + 'zizmor =1.29.0' \ 'cargo-criterion =1.1.0' \ 'critcmp =0.1.8' \ 'cargo-fuzz =0.13.2' \ @@ -571,12 +572,12 @@ mkdir -p "$cached_home/.cargo/bin" cat >"$cached_home/.cargo/bin/just" <<'SH' #!/usr/bin/env bash printf 'forged exact-version cache executed\n' >>"$MALICIOUS_EXEC_LOG" -printf 'just 1.57.0\n' +printf 'just 1.58.0\n' SH chmod +x "$cached_home/.cargo/bin/just" cat >"$cached_home/.cargo/.crates.toml" <<'EOF' [v1] -"just 1.57.0 (registry+https://github.com/rust-lang/crates.io-index)" = ["just"] +"just 1.58.0 (registry+https://github.com/rust-lang/crates.io-index)" = ["just"] EOF : >"$package_log" HOME="$cached_home" \ @@ -589,7 +590,7 @@ HOME="$cached_home" \ "$REPO_ROOT/scripts/ci/install-tools.sh" minimal >/dev/null [[ ! -e "$malicious_exec" ]] \ || fail "forged exact-version cached binary executed before authenticated replacement" -grep -Fq 'cargo install --registry crates-io just --locked --version =1.57.0 --force' "$package_log" \ +grep -Fq 'cargo install --registry crates-io just --locked --version =1.58.0 --force' "$package_log" \ || fail "forged cache did not trigger a fresh authenticated install" trusted_bin=$(tail -n 1 "$github_path_file") case "$trusted_bin" in @@ -600,11 +601,11 @@ esac : >"$package_log" MOCK_PACKAGE_LOG="$package_log" PATH="$package_bin:$PATH" \ "$REPO_ROOT/scripts/ci/setup-toolchain.sh" \ - nightly-2026-07-17 'clippy, rustfmt' >/dev/null + nightly-2026-08-12 'clippy, rustfmt' >/dev/null grep -Fq \ - 'rustup toolchain install nightly-2026-07-17 --profile minimal --no-self-update --component clippy --component rustfmt' \ + 'rustup toolchain install nightly-2026-08-12 --profile minimal --no-self-update --component clippy --component rustfmt' \ "$package_log" || fail "rustup toolchain command was not exact" -grep -Fq 'rustup default nightly-2026-07-17' "$package_log" \ +grep -Fq 'rustup default nightly-2026-08-12' "$package_log" \ || fail "rustup did not select the exact toolchain" if MOCK_PACKAGE_LOG="$package_log" PATH="$package_bin:$PATH" \ "$REPO_ROOT/scripts/ci/setup-toolchain.sh" nightly clippy >/dev/null 2>&1; then diff --git a/scripts/ct/dudect.sh b/scripts/ct/dudect.sh index 4c9a7a8a..cc454d62 100755 --- a/scripts/ct/dudect.sh +++ b/scripts/ct/dudect.sh @@ -115,7 +115,7 @@ BUILD_TARGET_DIR="$ROOT/target/ct-dudect-build/$TARGET/$PROFILE" if [[ ! -s "$LINKER_COMMAND_PATH" ]]; then rm -rf "$BUILD_TARGET_DIR" fi -CARGO_ARGS=(--locked --manifest-path "$ROOT/tools/ct-dudect/Cargo.toml" --target-dir "$BUILD_TARGET_DIR" --target "$TARGET") +CARGO_ARGS=(--manifest-path "$ROOT/tools/ct-dudect/Cargo.toml" --target-dir "$BUILD_TARGET_DIR" --target "$TARGET") if [[ "$PROFILE" == "release" ]]; then CARGO_ARGS+=(--release) elif [[ "$PROFILE" != "debug" ]]; then @@ -124,7 +124,7 @@ elif [[ "$PROFILE" != "debug" ]]; then fi linker_log_candidate="$(mktemp "$OUT_DIR/.dudect-linker-command.XXXXXXXX")" -cargo rustc "${CARGO_ARGS[@]}" --bin rscrypto-ct-dudect -- --print link-args 2>&1 | tee "$linker_log_candidate" +cargo rustc --locked "${CARGO_ARGS[@]}" --bin rscrypto-ct-dudect -- --print link-args 2>&1 | tee "$linker_log_candidate" link_command_count=$(grep -c '"-o"' "$linker_log_candidate" || true) if [[ "$link_command_count" -gt 1 ]]; then echo "expected at most one DudeCT linker command; found $link_command_count" >&2 diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 0387a3fc..e3b5ce49 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -158,7 +158,27 @@ apply_ci_resource_profile() { ensure_target() { local target=$1 - if ! rustup target list --installed 2>/dev/null | grep -q "^${target}$"; then + local toolchain=${2:-} + if [[ -n "$toolchain" ]]; then + if ! rustup target list --toolchain "$toolchain" --installed 2>/dev/null | grep -q "^${target}$"; then + rustup target add --toolchain "$toolchain" "$target" >/dev/null 2>&1 || true + fi + elif ! rustup target list --installed 2>/dev/null | grep -q "^${target}$"; then rustup target add "$target" >/dev/null 2>&1 || true fi } + +activate_nightly_toolchain() { + local toolchain_script="$COMMON_DIR/toolchain.sh" + local toolchain_contracts="$COMMON_DIR/../../.config/toolchains.toml" + [[ -x "$toolchain_script" ]] || { + echo "ERROR: nightly toolchain resolver not found: $toolchain_script" >&2 + return 1 + } + [[ -f "$toolchain_contracts" ]] || { + echo "ERROR: nightly toolchain authority not found: $toolchain_contracts" >&2 + return 1 + } + RUSTUP_TOOLCHAIN=$("$toolchain_script" --nightly) + export RUSTUP_TOOLCHAIN +} diff --git a/scripts/lib/fuzz-packages.sh b/scripts/lib/fuzz-packages.sh index c9bdf5dd..155a5ced 100644 --- a/scripts/lib/fuzz-packages.sh +++ b/scripts/lib/fuzz-packages.sh @@ -79,6 +79,9 @@ fuzz_in_package() { ( cd "$REPO_ROOT" || exit + if [[ "$subcommand" == build || "$subcommand" == run ]]; then + cargo metadata --locked --no-deps --manifest-path "$package_dir/Cargo.toml" >/dev/null + fi CARGO_TARGET_DIR="$FUZZ_SHARED_TARGET_DIR" cargo fuzz "$subcommand" --fuzz-dir "$package_dir" "$@" ) } diff --git a/scripts/lib/target-matrix.sh b/scripts/lib/target-matrix.sh index 18bda859..969a9100 100755 --- a/scripts/lib/target-matrix.sh +++ b/scripts/lib/target-matrix.sh @@ -25,21 +25,79 @@ if ! command -v jq >/dev/null 2>&1; then fi usage() { - echo "Usage: $0 --format {shell|json} [--key KEY]" >&2 + echo "Usage: $0 --validate | --format {shell|json} [--key KEY]" >&2 exit 1 } FORMAT="" KEY="" +VALIDATE=false while [[ $# -gt 0 ]]; do case "$1" in + --validate) VALIDATE=true; shift ;; --format) FORMAT="$2"; shift 2 ;; --key) KEY="$2"; shift 2 ;; *) usage ;; esac done +if [[ "$VALIDATE" == true && -n "$FORMAT" ]]; then + usage +fi + +validate_manifest() { + jq -e ' + . as $root | + ($root.groups | keys) == ["ibm", "linux", "macos", "no_std", "wasm", "win"] and + all($root.groups[]; type == "array" and length > 0 and . == sort and all(.[]; type == "string" and length > 0)) and + ([$root.groups[][]] | length) == ([$root.groups[][]] | unique | length) and + ($root.ci | type == "array" and length > 0) and + ($root.ci == ($root.ci | sort_by(.name))) and + ([$root.ci[].name] | length) == ([$root.ci[].name] | unique | length) and + all($root.ci[]; + .name as $name | + any($root.groups[][]; . == $name) and + if .type == "runson" then + (keys | sort) == ["name", "pool", "type"] + elif .type == "gha" then + (keys | sort) == ["name", "runner", "type"] + else + false + end + ) + ' "$MANIFEST" >/dev/null || { + echo "ERROR: invalid target matrix schema: $MANIFEST" >&2 + return 1 + } + + local matrix_targets + matrix_targets="$(jq -r '.groups[][]' "$MANIFEST" | LC_ALL=C sort)" + + local projection + for projection in "$REPO_ROOT/.config/rail.toml" "$REPO_ROOT/deny.toml"; do + local projected_targets + projected_targets="$(awk ' + /^targets = \[$/ { in_targets = 1; next } + in_targets && /^\]$/ { exit } + in_targets && match($0, /"[^"]+"/) { + print substr($0, RSTART + 1, RLENGTH - 2) + } + ' "$projection" | LC_ALL=C sort)" + if [[ "$projected_targets" != "$matrix_targets" ]]; then + echo "ERROR: target projection does not match .config/target-matrix.json: $projection" >&2 + diff -u <(printf '%s\n' "$matrix_targets") <(printf '%s\n' "$projected_targets") >&2 || true + return 1 + fi + done +} + +validate_manifest + +if [[ "$VALIDATE" == true ]]; then + exit 0 +fi + [[ -n "$FORMAT" ]] || usage print_shell() { diff --git a/scripts/lib/toolchain.sh b/scripts/lib/toolchain.sh index 9bc20554..48b7e881 100755 --- a/scripts/lib/toolchain.sh +++ b/scripts/lib/toolchain.sh @@ -2,24 +2,46 @@ # Extract toolchain channel from rust-toolchain.toml. # # Usage: -# scripts/lib/toolchain.sh # prints channel -# RUST_TOOLCHAIN=$(scripts/lib/toolchain.sh) +# scripts/lib/toolchain.sh # prints the development channel +# scripts/lib/toolchain.sh --msrv # prints the package MSRV +# scripts/lib/toolchain.sh --nightly # prints the pinned nightly channel set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" TOOLCHAIN_FILE="$REPO_ROOT/rust-toolchain.toml" +CARGO_MANIFEST="$REPO_ROOT/Cargo.toml" +TOOLCHAIN_CONTRACTS="$REPO_ROOT/.config/toolchains.toml" -if [ ! -f "$TOOLCHAIN_FILE" ]; then - echo "ERROR: rust-toolchain.toml not found at $TOOLCHAIN_FILE" >&2 +case "${1:-}" in + "") + SOURCE_FILE="$TOOLCHAIN_FILE" + FIELD_PATTERN='^channel[[:space:]]*=' + ;; + --msrv) + SOURCE_FILE="$CARGO_MANIFEST" + FIELD_PATTERN='^rust-version[[:space:]]*=' + ;; + --nightly) + SOURCE_FILE="$TOOLCHAIN_CONTRACTS" + FIELD_PATTERN='^nightly[[:space:]]*=' + ;; + *) + echo "Usage: $0 [--msrv|--nightly]" >&2 + exit 2 + ;; +esac + +if [[ ! -f "$SOURCE_FILE" ]]; then + echo "ERROR: toolchain authority not found: $SOURCE_FILE" >&2 exit 1 fi -RUST_TOOLCHAIN=$(awk -F'"' '/^channel/ {print $2}' "$TOOLCHAIN_FILE") +RUST_TOOLCHAIN=$(awk -F'"' -v pattern="$FIELD_PATTERN" '$0 ~ pattern { print $2; exit }' "$SOURCE_FILE") -if [ -z "$RUST_TOOLCHAIN" ]; then - echo "ERROR: Could not extract toolchain from $TOOLCHAIN_FILE" >&2 +if [[ -z "$RUST_TOOLCHAIN" ]]; then + echo "ERROR: could not extract toolchain from $SOURCE_FILE" >&2 exit 1 fi diff --git a/scripts/test/test-coverage.sh b/scripts/test/test-coverage.sh index deb772ab..b00d513e 100755 --- a/scripts/test/test-coverage.sh +++ b/scripts/test/test-coverage.sh @@ -79,7 +79,7 @@ cargo llvm-cov clean --workspace if [ "$RUN_NEXTEST" = true ]; then echo "━━━ Test Suite Coverage Capture (cargo-nextest) ━━━" - cargo llvm-cov nextest --no-report --workspace --all-features "${NEXTEST_ARGS[@]}" + cargo llvm-cov nextest --locked --no-report --workspace --all-features "${NEXTEST_ARGS[@]}" fi if [ "$RUN_FUZZ" = true ]; then @@ -87,14 +87,14 @@ if [ "$RUN_FUZZ" = true ]; then export RSCRYPTO_FUZZ_REPLAY_MISSING=skip echo "Full fuzz workspace replay" - cargo llvm-cov test --no-report \ + cargo llvm-cov test --locked --no-report \ --manifest-path "$REPO_ROOT/fuzz/Cargo.toml" \ --all-features --test corpus_replay -- --nocapture echo "Scoped fuzz package replay" while IFS= read -r manifest; do echo " -> ${manifest#"$REPO_ROOT"/}" - cargo llvm-cov test --no-report \ + cargo llvm-cov test --locked --no-report \ --manifest-path "$manifest" \ --all-features --test corpus_replay -- --nocapture done < <(find "$REPO_ROOT/fuzz-packages" -mindepth 2 -maxdepth 2 -name Cargo.toml | sort) diff --git a/scripts/test/test-fuzz-asan.sh b/scripts/test/test-fuzz-asan.sh index 06aac483..5f786ef6 100755 --- a/scripts/test/test-fuzz-asan.sh +++ b/scripts/test/test-fuzz-asan.sh @@ -11,6 +11,7 @@ source "$SCRIPT_DIR/../lib/common.sh" source "$SCRIPT_DIR/../lib/fuzz-packages.sh" maybe_disable_sccache +activate_nightly_toolchain PACKAGE_SCOPE="full" @@ -85,6 +86,7 @@ for package_dir in "${SELECTED_FUZZ_PACKAGES[@]:+${SELECTED_FUZZ_PACKAGES[@]}}"; echo "ASan corpus replay package: $(fuzz_package_label "$package_dir")" if ! cargo test \ + --locked \ -Zbuild-std \ --target "$(fuzz_host_target)" \ --manifest-path "$package_dir/Cargo.toml" \ diff --git a/scripts/test/test-fuzz-scheduler-test.sh b/scripts/test/test-fuzz-scheduler-test.sh index 2d1bce45..58a4f964 100755 --- a/scripts/test/test-fuzz-scheduler-test.sh +++ b/scripts/test/test-fuzz-scheduler-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" @@ -14,10 +15,12 @@ fail() { FIXTURE="$TMP_ROOT/repo" CAPTURE="$TMP_ROOT/capture" BIN="$TMP_ROOT/bin" -mkdir -p "$FIXTURE/scripts/test" "$FIXTURE/scripts/lib" "$FIXTURE/fuzz/corpus/alpha" "$CAPTURE" "$BIN" +mkdir -p "$FIXTURE/.config" "$FIXTURE/scripts/test" "$FIXTURE/scripts/lib" "$FIXTURE/fuzz/corpus/alpha" "$CAPTURE" "$BIN" cp "$REPO_ROOT/scripts/test/test-fuzz.sh" "$FIXTURE/scripts/test/" cp "$REPO_ROOT/scripts/lib/common.sh" "$REPO_ROOT/scripts/lib/rail-plan.sh" \ - "$REPO_ROOT/scripts/lib/fuzz-packages.sh" "$FIXTURE/scripts/lib/" + "$REPO_ROOT/scripts/lib/fuzz-packages.sh" "$REPO_ROOT/scripts/lib/toolchain.sh" \ + "$FIXTURE/scripts/lib/" +cp "$REPO_ROOT/.config/toolchains.toml" "$FIXTURE/.config/toolchains.toml" cat >"$FIXTURE/fuzz/Cargo.toml" <<'EOF' [package] diff --git a/scripts/test/test-fuzz.sh b/scripts/test/test-fuzz.sh index 32f824c7..fab0dded 100755 --- a/scripts/test/test-fuzz.sh +++ b/scripts/test/test-fuzz.sh @@ -29,6 +29,7 @@ source "$SCRIPT_DIR/../lib/common.sh" source "$SCRIPT_DIR/../lib/fuzz-packages.sh" maybe_disable_sccache +activate_nightly_toolchain # Configuration (can be overridden via environment) DURATION_SECS=${RSCRYPTO_FUZZ_DURATION_SECS:-60} diff --git a/scripts/test/test-miri.sh b/scripts/test/test-miri.sh index e17ddd00..a2619b37 100755 --- a/scripts/test/test-miri.sh +++ b/scripts/test/test-miri.sh @@ -34,6 +34,7 @@ echo "━━━━━━━━━━━━━━━━━━━━━━━━ echo "" maybe_disable_sccache +activate_nightly_toolchain unset RUSTC_WRAPPER unset CARGO_BUILD_RUSTC_WRAPPER @@ -87,7 +88,7 @@ run_miri_lib_filter() { echo "" echo "━━━ $label ━━━" - cargo miri test --lib --features "$MIRI_FEATURES" "$filter" + cargo miri test --locked --lib --features "$MIRI_FEATURES" "$filter" } run_miri_lib_filter_features() { @@ -100,14 +101,14 @@ run_miri_lib_filter_features() { echo "" echo "━━━ $label ━━━" - listing=$(cargo miri test --lib --features "$features" "$filter" -- --list) + listing=$(cargo miri test --locked --lib --features "$features" "$filter" -- --list) printf '%s\n' "$listing" count=$(printf '%s\n' "$listing" | awk '/: test$/ { count++ } END { print count + 0 }') if [ "$count" -ne "$expected_count" ]; then echo "Expected $expected_count Miri tests matching '$filter'; found $count" >&2 return 1 fi - cargo miri test --lib --features "$features" "$filter" + cargo miri test --locked --lib --features "$features" "$filter" } run_miri_test_target() { @@ -117,7 +118,7 @@ run_miri_test_target() { echo "" echo "━━━ $label ━━━" - cargo miri test --test "$target" --features "$features" + cargo miri test --locked --test "$target" --features "$features" } # Run Miri Tests @@ -152,7 +153,7 @@ case "$MIRI_SCOPE" in ;; exhaustive) echo "Scope: exhaustive lib tests under Miri" - cargo miri test --lib --features "$MIRI_FEATURES" + cargo miri test --locked --lib --features "$MIRI_FEATURES" ;; *) echo "Invalid RSCRYPTO_MIRI_SCOPE: $MIRI_SCOPE" diff --git a/scripts/test/test-rsa-leakage.sh b/scripts/test/test-rsa-leakage.sh index 87207a34..27294421 100755 --- a/scripts/test/test-rsa-leakage.sh +++ b/scripts/test/test-rsa-leakage.sh @@ -21,7 +21,7 @@ echo "Welch t threshold: $RSCRYPTO_RSA_LEAKAGE_T_THRESHOLD" echo "Warmup iterations: $RSCRYPTO_RSA_LEAKAGE_WARMUP" echo "" -cargo test --release --test rsa_leakage --features rsa,diag,getrandom \ +cargo test --locked --release --test rsa_leakage --features rsa,diag,getrandom \ rsa_private_operations_do_not_show_first_order_timing_leakage \ -- --ignored --nocapture diff --git a/scripts/test/test.sh b/scripts/test/test.sh index 1f52f7b6..593ad75f 100755 --- a/scripts/test/test.sh +++ b/scripts/test/test.sh @@ -57,7 +57,7 @@ run_workspace_doctests() { echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Running doctests for entire workspace" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - cargo test --workspace --doc --all-features + cargo test --locked --workspace --doc --all-features } run_crate_doctests() { @@ -77,7 +77,7 @@ run_crate_doctests() { echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" for crate in "${crates[@]}"; do - cargo test -p "$crate" --doc --all-features + cargo test --locked -p "$crate" --doc --all-features done } @@ -135,21 +135,21 @@ if [ ${#CRATES[@]} -gt 0 ]; then echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" if [ "$HAS_NEXTEST" = true ]; then # shellcheck disable=SC2086 - cargo nextest run $CRATE_FLAGS -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" + cargo nextest run --locked $CRATE_FLAGS -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" run_crate_doctests "${CRATES[@]}" else # shellcheck disable=SC2086 - cargo test $CRATE_FLAGS --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" + cargo test --locked $CRATE_FLAGS --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" fi elif [ "$ALL_FLAG" = true ]; then echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Testing entire workspace" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" if [ "$HAS_NEXTEST" = true ]; then - cargo nextest run --workspace -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" + cargo nextest run --locked --workspace -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" run_workspace_doctests else - cargo test --workspace --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" + cargo test --locked --workspace --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" fi else # Rail-scoped (default): cargo-rail planner selects the affected crates. @@ -173,11 +173,11 @@ else for crate in $affected; do CRATE_FLAGS+=(-p "$crate") done - cargo nextest run "${CRATE_FLAGS[@]}" -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" + cargo nextest run --locked "${CRATE_FLAGS[@]}" -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" run_rail_scoped_doctests else for crate in $affected; do - cargo test -p "$crate" --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" + cargo test --locked -p "$crate" --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" done fi exit 0 @@ -186,9 +186,9 @@ else esac if [ "$HAS_NEXTEST" = true ]; then - cargo nextest run --workspace -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" + cargo nextest run --locked --workspace -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" run_workspace_doctests else - cargo test --workspace --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" + cargo test --locked --workspace --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" fi fi diff --git a/src/aead/aegis256.rs b/src/aead/aegis256.rs index 849441d5..efc6c093 100644 --- a/src/aead/aegis256.rs +++ b/src/aead/aegis256.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! AEGIS-256 authenticated encryption (draft-irtf-cfrg-aegis-aead). //! //! AES-round-based AEAD with a 256-bit key, 256-bit nonce, and 128-bit @@ -16,6 +14,7 @@ use core::fmt; ))] use super::targets::AeadBackend; #[cfg(any( + target_arch = "x86_64", target_arch = "aarch64", all(target_arch = "powerpc64", target_endian = "little"), target_arch = "riscv64", @@ -217,31 +216,21 @@ fn finalize(s: &mut State, ad_len: usize, msg_len: usize) -> [u8; TAG_SIZE] { // riscv64 scalar AES backend (Zkne) #[cfg(target_arch = "aarch64")] -#[allow(unsafe_op_in_unsafe_fn)] #[path = "aegis256/aarch64_ce.rs"] mod ce; #[cfg(target_arch = "x86_64")] -#[allow(unsafe_op_in_unsafe_fn)] #[path = "aegis256/x86_64_ni.rs"] mod ni; #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] -#[allow(unsafe_code, unsafe_op_in_unsafe_fn)] #[path = "aegis256/powerpc64_ppc.rs"] mod ppc; #[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] -#[path = "aegis256/riscv64_vperm.rs"] -mod rv_vperm; -#[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] #[path = "aegis256/riscv64_zkne.rs"] mod rv_zkne; #[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] #[path = "aegis256/riscv64_zvkned.rs"] mod rv_zvkned; #[cfg(target_arch = "s390x")] -#[allow(unsafe_code)] #[path = "aegis256/s390x_vperm.rs"] mod s390x_vperm; #[cfg(any( @@ -254,20 +243,7 @@ mod s390x_vperm; #[inline] fn resolve_backend() -> AeadBackend { let caps = crate::platform::caps(); - - #[cfg(target_arch = "x86_64")] - { - use crate::platform::caps::x86; - if caps.has(x86::AESNI) && caps.has(x86::AVX) { - return AeadBackend::X86Aesni; - } - AeadBackend::Portable - } - - #[cfg(not(target_arch = "x86_64"))] - { - select_backend(AeadPrimitive::Aegis256, crate::platform::arch(), caps) - } + select_backend(AeadPrimitive::Aegis256, crate::platform::arch(), caps) } // Key @@ -435,6 +411,7 @@ fn encrypt_portable(key: &[u8; KEY_SIZE], nonce: &[u8; NONCE_SIZE], aad: &[u8], } #[cfg(feature = "diag")] +/// Run one portable AEGIS-256 state update and return its diagnostic final tag. #[unsafe(no_mangle)] #[inline(never)] #[must_use] @@ -570,13 +547,6 @@ impl Aead for Aegis256 { return Ok(Aegis256Tag::from_bytes(tag)); } - #[cfg(target_arch = "riscv64")] - if self.backend == AeadBackend::Riscv64Vperm { - // SAFETY: backend resolution confirmed the RISC-V V extension is available. - let tag = unsafe { rv_vperm::encrypt_fused(key, nonce, aad, buffer) }; - return Ok(Aegis256Tag::from_bytes(tag)); - } - #[cfg(target_arch = "s390x")] if self.backend == AeadBackend::S390xVperm { // SAFETY: s390x vperm backend call because: @@ -584,7 +554,6 @@ impl Aead for Aegis256 { // 2. `select_backend` returns `S390xVperm` only when caps include `s390x::VECTOR`. // 3. `key`, `nonce`, `aad`, and `buffer` are valid references from the safe AEAD API. let tag = unsafe { s390x_vperm::encrypt_fused(key, nonce, aad, buffer) }; - #[allow(clippy::needless_return)] return Ok(Aegis256Tag::from_bytes(tag)); } @@ -635,9 +604,6 @@ impl Aead for Aegis256 { } else if self.backend == AeadBackend::Riscv64ScalarCrypto { // SAFETY: backend resolution confirmed scalar AES (`zkne`) is available. unsafe { rv_zkne::decrypt_fused(key, nonce, aad, buffer) } - } else if self.backend == AeadBackend::Riscv64Vperm { - // SAFETY: backend resolution confirmed the RISC-V V extension is available. - unsafe { rv_vperm::decrypt_fused(key, nonce, aad, buffer) } } else { decrypt_portable(key, nonce, aad, buffer) }; @@ -676,10 +642,12 @@ impl Aead for Aegis256 { #[cfg(test)] mod tests { use alloc::{vec, vec::Vec}; - use std::eprintln; use super::*; - use crate::aead::expert::AeadWithNonce; + use crate::aead::{ + expert::AeadWithNonce, + test_vectors::{hex_array, hex_vec as hex}, + }; #[cfg(not(target_arch = "s390x"))] #[inline(always)] @@ -687,18 +655,8 @@ mod tests { super::super::aes_round::aes_enc_round_portable(block, round_key) } - fn hex(s: &str) -> Vec { - (0..s.len()) - .step_by(2) - .map(|i| u8::from_str_radix(&s[i..i.strict_add(2)], 16).unwrap()) - .collect() - } - fn hex_block(s: &str) -> [u8; 16] { - let v = hex(s); - let mut out = [0u8; 16]; - out.copy_from_slice(&v); - out + hex_array(s) } // -- AESRound test vector (Appendix A.1) -- @@ -767,15 +725,10 @@ mod tests { // -- Spec test vectors (Appendix A.3) -- fn spec_key() -> Aegis256Key { - Aegis256Key::from_bytes( - hex_block("10010000000000000000000000000000") - .iter() - .chain(hex_block("00000000000000000000000000000000").iter()) - .copied() - .collect::>() - .try_into() - .unwrap(), - ) + Aegis256Key::from_bytes(hex_array(concat!( + "10010000000000000000000000000000", + "00000000000000000000000000000000" + ))) } fn spec_nonce() -> Nonce256 { @@ -793,12 +746,16 @@ mod tests { // Encrypt. let mut buf = msg.to_vec(); - let tag = aead.encrypt_in_place(&nonce, aad, &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, aad, &mut buf) + .expect("AEGIS-256 specification-vector encryption must succeed"); assert_eq!(&buf, &expected_ct, "ciphertext mismatch"); assert_eq!(tag.as_bytes(), expected_tag.as_slice(), "tag mismatch"); // Decrypt round-trip. - aead.decrypt_in_place(&nonce, aad, &mut buf, &tag).unwrap(); + aead + .decrypt_in_place(&nonce, aad, &mut buf, &tag) + .expect("AEGIS-256 specification-vector decryption must succeed"); assert_eq!(&buf, msg, "plaintext recovery mismatch"); } @@ -861,8 +818,12 @@ mod tests { let aead = Aegis256::new(&key); let mut buf = []; - let tag = aead.encrypt_in_place(&nonce, b"", &mut buf).unwrap(); - aead.decrypt_in_place(&nonce, b"", &mut buf, &tag).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"", &mut buf) + .expect("empty AEGIS-256 encryption must succeed"); + aead + .decrypt_in_place(&nonce, b"", &mut buf, &tag) + .expect("empty AEGIS-256 decryption must succeed"); } #[test] @@ -873,10 +834,14 @@ mod tests { let plaintext = b"the quick brown fox jumps over the lazy dog"; let mut buf = *plaintext; - let tag = aead.encrypt_in_place(&nonce, b"header", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"header", &mut buf) + .expect("AEGIS-256 encryption with AAD must succeed"); assert_ne!(&buf[..], &plaintext[..]); - aead.decrypt_in_place(&nonce, b"header", &mut buf, &tag).unwrap(); + aead + .decrypt_in_place(&nonce, b"header", &mut buf, &tag) + .expect("AEGIS-256 decryption with AAD must succeed"); assert_eq!(&buf[..], &plaintext[..]); } @@ -889,10 +854,10 @@ mod tests { let mut buf = []; let tag = aead .encrypt_in_place(&nonce, b"associated data only", &mut buf) - .unwrap(); + .expect("AAD-only AEGIS-256 encryption must succeed"); aead .decrypt_in_place(&nonce, b"associated data only", &mut buf, &tag) - .unwrap(); + .expect("AAD-only AEGIS-256 decryption must succeed"); } #[test] @@ -902,14 +867,18 @@ mod tests { let aead = Aegis256::new(&key); let mut buf = *b"zero me on failure"; - let tag = aead.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AEGIS-256 test setup encryption must succeed"); let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0xFF; let bad_tag = Aegis256Tag::from_bytes(bad_tag); - let result = aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert!(buf.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } @@ -920,11 +889,15 @@ mod tests { let aead = Aegis256::new(&key); let mut buf = *b"secret"; - let tag = aead.encrypt_in_place(&nonce, b"", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"", &mut buf) + .expect("AEGIS-256 test setup encryption must succeed"); buf[0] ^= 1; - let result = aead.decrypt_in_place(&nonce, b"", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"", &mut buf, &tag), + Err(OpenError::verification()) + ); assert_eq!(&buf, &[0u8; 6]); } @@ -935,14 +908,18 @@ mod tests { let aead = Aegis256::new(&key); let mut buf = *b"data"; - let tag = aead.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AEGIS-256 test setup encryption must succeed"); let mut bad_tag_bytes = tag.to_bytes(); bad_tag_bytes[15] ^= 1; let bad_tag = Aegis256Tag::from_bytes(bad_tag_bytes); - let result = aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert_eq!(&buf, &[0u8; 4]); } @@ -953,10 +930,14 @@ mod tests { let aead = Aegis256::new(&key); let mut buf = *b"msg"; - let tag = aead.encrypt_in_place(&nonce, b"correct", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"correct", &mut buf) + .expect("AEGIS-256 test setup encryption must succeed"); - let result = aead.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag), + Err(OpenError::verification()) + ); } #[test] @@ -966,11 +947,15 @@ mod tests { let aead = Aegis256::new(&key); let mut buf = *b"nonce test"; - let tag = aead.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AEGIS-256 test setup encryption must succeed"); let wrong_nonce = Nonce256::from_bytes([11; 32]); - let result = aead.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } #[test] @@ -981,18 +966,29 @@ mod tests { let pt = b"combined mode"; let mut sealed = vec![0u8; pt.len().strict_add(TAG_SIZE)]; - aead.encrypt(&nonce, b"h", pt.as_slice(), &mut sealed).unwrap(); + aead + .encrypt(&nonce, b"h", pt.as_slice(), &mut sealed) + .expect("combined AEGIS-256 encryption must succeed"); let mut opened = vec![0u8; pt.len()]; - aead.decrypt(&nonce, b"h", &sealed, &mut opened).unwrap(); + aead + .decrypt(&nonce, b"h", &sealed, &mut opened) + .expect("combined AEGIS-256 decryption must succeed"); assert_eq!(&opened, &pt[..]); } #[test] fn tag_from_slice_rejects_wrong_length() { - assert!(Aegis256::tag_from_slice(&[0u8; 15]).is_err()); - assert!(Aegis256::tag_from_slice(&[0u8; 17]).is_err()); - assert!(Aegis256::tag_from_slice(&[0u8; 16]).is_ok()); + assert_eq!( + Aegis256::tag_from_slice(&[0u8; 15]).expect_err("short AEGIS-256 tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aegis256::tag_from_slice(&[0u8; 17]).expect_err("long AEGIS-256 tag must be rejected"), + AeadBufferError::new() + ); + let tag = Aegis256::tag_from_slice(&[0u8; 16]).expect("16-byte AEGIS-256 tag must be accepted"); + assert_eq!(tag.as_bytes(), &[0u8; 16]); } #[test] @@ -1006,7 +1002,7 @@ mod tests { let mut buf = plaintext; let tag = aead .encrypt_in_place(&nonce, b"multi-block aad that is longer than one rate block", &mut buf) - .unwrap(); + .expect("multi-block AEGIS-256 encryption must succeed"); aead .decrypt_in_place( &nonce, @@ -1014,7 +1010,7 @@ mod tests { &mut buf, &tag, ) - .unwrap(); + .expect("multi-block AEGIS-256 decryption must succeed"); assert_eq!(buf, plaintext); } @@ -1027,15 +1023,23 @@ mod tests { // Exactly 16 bytes = 1 full block, 0-byte tail. let plaintext = [0x55u8; 16]; let mut buf = plaintext; - let tag = aead.encrypt_in_place(&nonce, b"", &mut buf).unwrap(); - aead.decrypt_in_place(&nonce, b"", &mut buf, &tag).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"", &mut buf) + .expect("one-block AEGIS-256 encryption must succeed"); + aead + .decrypt_in_place(&nonce, b"", &mut buf, &tag) + .expect("one-block AEGIS-256 decryption must succeed"); assert_eq!(buf, plaintext); // Exactly 32 bytes = 2 full blocks, 0-byte tail. let plaintext32 = [0x66u8; 32]; let mut buf32 = plaintext32; - let tag32 = aead.encrypt_in_place(&nonce, b"", &mut buf32).unwrap(); - aead.decrypt_in_place(&nonce, b"", &mut buf32, &tag32).unwrap(); + let tag32 = aead + .encrypt_in_place(&nonce, b"", &mut buf32) + .expect("two-block AEGIS-256 encryption must succeed"); + aead + .decrypt_in_place(&nonce, b"", &mut buf32, &tag32) + .expect("two-block AEGIS-256 decryption must succeed"); assert_eq!(buf32, plaintext32); } @@ -1047,12 +1051,16 @@ mod tests { let aead = Aegis256::new(&key); let aad = b"four-block-test"; - for &size in &[48, 64, 80, 96, 112, 128, 256, 1024, 4096] { - let plaintext: Vec = (0..size).map(|i| (i & 0xFF) as u8).collect(); + for &size in &[48usize, 64, 80, 96, 112, 128, 256, 1024, 4096] { + let plaintext: Vec = (0u8..=u8::MAX).cycle().take(size).collect(); let mut buf = plaintext.clone(); - let tag = aead.encrypt_in_place(&nonce, aad, &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, aad, &mut buf) + .expect("boundary AEGIS-256 encryption must succeed"); assert_ne!(&buf, &plaintext, "size {size}: ciphertext must differ"); - aead.decrypt_in_place(&nonce, aad, &mut buf, &tag).unwrap(); + aead + .decrypt_in_place(&nonce, aad, &mut buf, &tag) + .expect("boundary AEGIS-256 decryption must succeed"); assert_eq!(&buf, &plaintext, "size {size}: round-trip failed"); } } @@ -1066,7 +1074,7 @@ mod tests { /// Scalar simulation of vperm: table[index & 0x0F]. fn vperm_scalar(table: &[u8; 16], index: u8) -> u8 { - table[(index & 0x0F) as usize] + table[usize::from(index & 0x0F)] } /// Scalar simulation of vperm with PSHUFB zeroing: returns 0 when bit 7 set. @@ -1074,7 +1082,7 @@ mod tests { if index & 0x80 != 0 { 0 } else { - table[(index & 0x0F) as usize] + table[usize::from(index & 0x0F)] } } @@ -1147,24 +1155,14 @@ mod tests { 0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16, ]; - let mut failures = 0u32; - for input in 0u16..256 { - let got = vperm_sbox_scalar(input as u8); + for input in u8::MIN..=u8::MAX { + let got = vperm_sbox_scalar(input); // The Hamburg vperm S-box omits the AES affine constant. // vpaes_sbox(x) = AES_sbox(x) ^ AES_AFFINE for all x. use super::super::aes_round::AES_AFFINE; - let expected = AES_SBOX[input as usize] ^ AES_AFFINE; - if got != expected { - if failures < 16 { - eprintln!( - "vperm S-box mismatch at input 0x{:02X}: got 0x{:02X}, expected 0x{:02X}", - input, got, expected, - ); - } - failures = failures.strict_add(1); - } + let expected = AES_SBOX[usize::from(input)] ^ AES_AFFINE; + assert_eq!(got, expected, "vperm S-box mismatch at input 0x{input:02X}"); } - assert_eq!(failures, 0, "{failures} vperm S-box mismatches out of 256"); } // -- Full vperm AES round validation -- @@ -1180,35 +1178,37 @@ mod tests { // SubBytes via vperm tower field (includes affine constant compensation) use super::super::aes_round::{AES_AFFINE, VPERM_SR as SR}; let mut sb = [0u8; 16]; - for i in 0..16 { - sb[i] = vperm_sbox_scalar(block[i]) ^ AES_AFFINE; + for (substituted, &input) in sb.iter_mut().zip(block) { + *substituted = vperm_sbox_scalar(input) ^ AES_AFFINE; } // ShiftRows let mut sr = [0u8; 16]; - for i in 0..16 { - sr[i] = sb[SR[i] as usize]; + for (shifted, &source) in sr.iter_mut().zip(&SR) { + *shifted = sb[usize::from(source)]; } // MixColumns via xtime decomposition fn xtime(b: u8) -> u8 { - let r = (b as u16) << 1; - (r ^ (if r & 0x100 != 0 { 0x1B } else { 0 })) as u8 + let doubled = u16::from(b).strict_mul(2); + let reduced = if doubled & 0x100 == 0 { doubled } else { doubled ^ 0x11B }; + u8::try_from(reduced).expect("AES polynomial reduction must produce one byte") } let mut mc = [0u8; 16]; - for col in 0..4 { - let c = col * 4; - let (b0, b1, b2, b3) = (sr[c], sr[c + 1], sr[c + 2], sr[c + 3]); - mc[c] = xtime(b0) ^ xtime(b1) ^ b1 ^ b2 ^ b3; - mc[c + 1] = b0 ^ xtime(b1) ^ xtime(b2) ^ b2 ^ b3; - mc[c + 2] = b0 ^ b1 ^ xtime(b2) ^ xtime(b3) ^ b3; - mc[c + 3] = xtime(b0) ^ b0 ^ b1 ^ b2 ^ xtime(b3); + for (source, mixed) in sr.as_chunks::<4>().0.iter().zip(mc.as_chunks_mut::<4>().0) { + let [b0, b1, b2, b3] = *source; + *mixed = [ + xtime(b0) ^ xtime(b1) ^ b1 ^ b2 ^ b3, + b0 ^ xtime(b1) ^ xtime(b2) ^ b2 ^ b3, + b0 ^ b1 ^ xtime(b2) ^ xtime(b3) ^ b3, + xtime(b0) ^ b0 ^ b1 ^ b2 ^ xtime(b3), + ]; } // AddRoundKey let mut result = [0u8; 16]; - for i in 0..16 { - result[i] = mc[i] ^ round_key[i]; + for ((output, &mixed), &key_byte) in result.iter_mut().zip(&mc).zip(round_key) { + *output = mixed ^ key_byte; } result } @@ -1237,9 +1237,9 @@ mod tests { assert_eq!(vperm_aes_round_scalar(&a, &b), portable_aes_round(&a, &b)); // Exhaustive: test all single-byte patterns in position 0 - for val in 0u16..256 { + for val in u8::MIN..=u8::MAX { let mut block = [0u8; 16]; - block[0] = val as u8; + block[0] = val; let key = [0u8; 16]; let got = vperm_aes_round_scalar(&block, &key); let expected = portable_aes_round(&block, &key); diff --git a/src/aead/aegis256/aarch64_ce.rs b/src/aead/aegis256/aarch64_ce.rs index d8262b58..2809fd1d 100644 --- a/src/aead/aegis256/aarch64_ce.rs +++ b/src/aead/aegis256/aarch64_ce.rs @@ -4,18 +4,26 @@ use super::{BLOCK_SIZE, C0, C1, KEY_SIZE, NONCE_SIZE, TAG_SIZE}; #[inline] /// # Safety -/// -/// `bytes` must refer to a valid 16-byte block. +/// Caller must ensure `neon` support is available. unsafe fn load(bytes: &[u8; BLOCK_SIZE]) -> uint8x16_t { - vld1q_u8(bytes.as_ptr()) + // SAFETY: the caller guarantees NEON; `bytes` provides 16 initialized bytes for this unaligned load. + unsafe { vld1q_u8(bytes.as_ptr()) } } #[inline] /// # Safety -/// -/// `out` must refer to a valid writable 16-byte block. +/// Caller must ensure `neon` support is available. unsafe fn store(v: uint8x16_t, out: &mut [u8; BLOCK_SIZE]) { - vst1q_u8(out.as_mut_ptr(), v); + // SAFETY: the caller guarantees NEON; `out` provides exclusive access to 16 bytes for this unaligned store. + unsafe { vst1q_u8(out.as_mut_ptr(), v) }; +} + +#[inline(always)] +fn prefetch_read_l1(ptr: *const u8) { + // SAFETY: `prfm` is an AArch64 cache hint that does not dereference `ptr` or alter architectural state. + unsafe { + core::arch::asm!("prfm pldl1keep, [{ptr}]", ptr = in(reg) ptr, options(nostack, preserves_flags)); + } } // ── Register-based helpers ────────────────────────────────────────────── @@ -27,7 +35,6 @@ unsafe fn store(v: uint8x16_t, out: &mut [u8; BLOCK_SIZE]) { #[target_feature(enable = "aes,neon")] #[inline] -#[allow(clippy::too_many_arguments)] /// # Safety /// /// Caller must ensure `aes` and `neon` support is available and all register @@ -40,8 +47,8 @@ unsafe fn update_regs( s4: &mut uint8x16_t, s5: &mut uint8x16_t, m: uint8x16_t, - zero: uint8x16_t, ) { + let zero = vdupq_n_u8(0); let tmp = *s5; #[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos"))] @@ -68,10 +75,10 @@ unsafe fn update_regs( #[inline] /// # Safety /// -/// The provided registers must come from a valid AEGIS-256 state on an -/// `aes` + `neon` capable CPU. +/// Caller must ensure `neon` support is available. unsafe fn keystream_regs(s1: uint8x16_t, s2: uint8x16_t, s3: uint8x16_t, s4: uint8x16_t, s5: uint8x16_t) -> uint8x16_t { - veorq_u8(veorq_u8(s1, s4), veorq_u8(s5, vandq_u8(s2, s3))) + // SAFETY: the caller guarantees NEON support for these register-only operations. + unsafe { veorq_u8(veorq_u8(s1, s4), veorq_u8(s5, vandq_u8(s2, s3))) } } // ── Fused encrypt/decrypt ─────────────────────────────────────────────── @@ -88,41 +95,54 @@ pub(super) unsafe fn encrypt_fused( ) -> [u8; TAG_SIZE] { let (kh0, kh1) = super::split_halves(key); let (nh0, nh1) = super::split_halves(nonce); - let k0 = load(kh0); - let k1 = load(kh1); - let n0 = load(nh0); - let n1 = load(nh1); - let c0 = load(&C0); - let c1 = load(&C1); + // SAFETY: the entrypoint enables NEON; `kh0` is a complete initialized block. + let k0 = unsafe { load(kh0) }; + // SAFETY: the entrypoint enables NEON; `kh1` is a complete initialized block. + let k1 = unsafe { load(kh1) }; + // SAFETY: the entrypoint enables NEON; `nh0` is a complete initialized block. + let n0 = unsafe { load(nh0) }; + // SAFETY: the entrypoint enables NEON; `nh1` is a complete initialized block. + let n1 = unsafe { load(nh1) }; + // SAFETY: the entrypoint enables NEON; `C0` is a complete initialized block. + let c0 = unsafe { load(&C0) }; + // SAFETY: the entrypoint enables NEON; `C1` is a complete initialized block. + let c1 = unsafe { load(&C1) }; let k0_xor_n0 = veorq_u8(k0, n0); let k1_xor_n1 = veorq_u8(k1, n1); - let zero = vdupq_n_u8(0); let (mut s0, mut s1, mut s2, mut s3, mut s4, mut s5) = (k0_xor_n0, k1_xor_n1, c1, c0, veorq_u8(k0, c0), veorq_u8(k1, c1)); for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0, zero); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1, zero); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0, zero); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1, zero); + // SAFETY: the entrypoint establishes AES and NEON for these valid state registers. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1); + } } let mut offset = 0usize; while offset.strict_add(BLOCK_SIZE) <= aad.len() { - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - vld1q_u8(aad.as_ptr().add(offset)), - zero, - ); + // SAFETY: the loop bound provides a complete readable block at `offset`; the entrypoint enables AES and NEON. + unsafe { + update_regs( + &mut s0, + &mut s1, + &mut s2, + &mut s3, + &mut s4, + &mut s5, + vld1q_u8(aad.as_ptr().add(offset)), + ); + } offset = offset.strict_add(BLOCK_SIZE); } if offset < aad.len() { let mut pad = [0u8; BLOCK_SIZE]; pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pad), zero); + // SAFETY: `pad` is a complete initialized block, and the entrypoint enables AES and NEON. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pad)); + } } let msg_len = buffer.len(); let ptr = buffer.as_mut_ptr(); @@ -131,55 +151,71 @@ pub(super) unsafe fn encrypt_fused( let four_blocks = BLOCK_SIZE.strict_mul(4); let two_blocks = BLOCK_SIZE.strict_mul(2); while offset.strict_add(four_blocks) <= len { - core::arch::asm!("prfm pldl1keep, [{ptr}]", ptr = in(reg) ptr.add(offset.strict_add(192)), options(nostack, preserves_flags)); - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let xi_a = vld1q_u8(ptr.add(offset)); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a, zero); - vst1q_u8(ptr.add(offset), veorq_u8(xi_a, z_a)); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let xi_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b, zero); - vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), veorq_u8(xi_b, z_b)); - let z_c = keystream_regs(s1, s2, s3, s4, s5); - let xi_c = vld1q_u8(ptr.add(offset.strict_add(two_blocks))); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c, zero); - vst1q_u8(ptr.add(offset.strict_add(two_blocks)), veorq_u8(xi_c, z_c)); - let z_d = keystream_regs(s1, s2, s3, s4, s5); - let xi_d = vld1q_u8(ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE)))); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d, zero); - vst1q_u8( - ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE))), - veorq_u8(xi_d, z_d), - ); + prefetch_read_l1(ptr.wrapping_add(offset.strict_add(192))); + // SAFETY: the loop bound places all four 16-byte lanes inside the exclusively borrowed buffer. The entrypoint + // enables AES and NEON, and each plaintext lane is loaded before its ciphertext overwrites the same bytes. + unsafe { + let z_a = keystream_regs(s1, s2, s3, s4, s5); + let xi_a = vld1q_u8(ptr.add(offset)); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); + vst1q_u8(ptr.add(offset), veorq_u8(xi_a, z_a)); + let z_b = keystream_regs(s1, s2, s3, s4, s5); + let xi_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); + vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), veorq_u8(xi_b, z_b)); + let z_c = keystream_regs(s1, s2, s3, s4, s5); + let xi_c = vld1q_u8(ptr.add(offset.strict_add(two_blocks))); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c); + vst1q_u8(ptr.add(offset.strict_add(two_blocks)), veorq_u8(xi_c, z_c)); + let z_d = keystream_regs(s1, s2, s3, s4, s5); + let xi_d = vld1q_u8(ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE)))); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d); + vst1q_u8( + ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE))), + veorq_u8(xi_d, z_d), + ); + } offset = offset.strict_add(four_blocks); } if offset.strict_add(two_blocks) <= len { - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let xi_a = vld1q_u8(ptr.add(offset)); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a, zero); - vst1q_u8(ptr.add(offset), veorq_u8(xi_a, z_a)); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let xi_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b, zero); - vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), veorq_u8(xi_b, z_b)); + // SAFETY: the branch bound places both lanes inside the exclusively borrowed buffer; AES and NEON are enabled. + unsafe { + let z_a = keystream_regs(s1, s2, s3, s4, s5); + let xi_a = vld1q_u8(ptr.add(offset)); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); + vst1q_u8(ptr.add(offset), veorq_u8(xi_a, z_a)); + let z_b = keystream_regs(s1, s2, s3, s4, s5); + let xi_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); + vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), veorq_u8(xi_b, z_b)); + } offset = offset.strict_add(two_blocks); } if offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream_regs(s1, s2, s3, s4, s5); - let xi = vld1q_u8(ptr.add(offset)); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi, zero); - vst1q_u8(ptr.add(offset), veorq_u8(xi, z)); + // SAFETY: the branch bound provides one complete writable lane in the buffer; AES and NEON are enabled. + unsafe { + let z = keystream_regs(s1, s2, s3, s4, s5); + let xi = vld1q_u8(ptr.add(offset)); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); + vst1q_u8(ptr.add(offset), veorq_u8(xi, z)); + } offset = offset.strict_add(BLOCK_SIZE); } if offset < len { - let z = keystream_regs(s1, s2, s3, s4, s5); + // SAFETY: the entrypoint enables NEON for the valid state registers. + let z = unsafe { keystream_regs(s1, s2, s3, s4, s5) }; let tail_len = len.strict_sub(offset); let mut pad = [0u8; BLOCK_SIZE]; pad[..tail_len].copy_from_slice(&buffer[offset..]); - let xi = load(&pad); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi, zero); + // SAFETY: `pad` is a complete initialized block, and the entrypoint enables NEON. + let xi = unsafe { load(&pad) }; + // SAFETY: the entrypoint establishes AES and NEON for these valid state registers. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); + } let mut ct_bytes = [0u8; BLOCK_SIZE]; - store(veorq_u8(xi, z), &mut ct_bytes); + // SAFETY: `ct_bytes` is a complete writable block, and the entrypoint enables NEON. + unsafe { store(veorq_u8(xi, z), &mut ct_bytes) }; buffer[offset..].copy_from_slice(&ct_bytes[..tail_len]); } let ad_bits = (aad.len() as u64).strict_mul(8); @@ -187,13 +223,18 @@ pub(super) unsafe fn encrypt_fused( let mut len_bytes = [0u8; BLOCK_SIZE]; len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); len_bytes[8..].copy_from_slice(&msg_bits.to_le_bytes()); - let t = veorq_u8(s3, load(&len_bytes)); + // SAFETY: `len_bytes` is a complete initialized block, and the entrypoint enables NEON. + let t = unsafe { veorq_u8(s3, load(&len_bytes)) }; for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t, zero); + // SAFETY: the entrypoint establishes AES and NEON for these valid state registers. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t); + } } let tag_vec = veorq_u8(veorq_u8(veorq_u8(s0, s1), veorq_u8(s2, s3)), veorq_u8(s4, s5)); let mut tag = [0u8; TAG_SIZE]; - store(tag_vec, &mut tag); + // SAFETY: `tag` is a complete writable block, and the entrypoint enables NEON. + unsafe { store(tag_vec, &mut tag) }; tag } @@ -209,41 +250,54 @@ pub(super) unsafe fn decrypt_fused( ) -> [u8; TAG_SIZE] { let (kh0, kh1) = super::split_halves(key); let (nh0, nh1) = super::split_halves(nonce); - let k0 = load(kh0); - let k1 = load(kh1); - let n0 = load(nh0); - let n1 = load(nh1); - let c0 = load(&C0); - let c1 = load(&C1); + // SAFETY: the entrypoint enables NEON; `kh0` is a complete initialized block. + let k0 = unsafe { load(kh0) }; + // SAFETY: the entrypoint enables NEON; `kh1` is a complete initialized block. + let k1 = unsafe { load(kh1) }; + // SAFETY: the entrypoint enables NEON; `nh0` is a complete initialized block. + let n0 = unsafe { load(nh0) }; + // SAFETY: the entrypoint enables NEON; `nh1` is a complete initialized block. + let n1 = unsafe { load(nh1) }; + // SAFETY: the entrypoint enables NEON; `C0` is a complete initialized block. + let c0 = unsafe { load(&C0) }; + // SAFETY: the entrypoint enables NEON; `C1` is a complete initialized block. + let c1 = unsafe { load(&C1) }; let k0_xor_n0 = veorq_u8(k0, n0); let k1_xor_n1 = veorq_u8(k1, n1); - let zero = vdupq_n_u8(0); let (mut s0, mut s1, mut s2, mut s3, mut s4, mut s5) = (k0_xor_n0, k1_xor_n1, c1, c0, veorq_u8(k0, c0), veorq_u8(k1, c1)); for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0, zero); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1, zero); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0, zero); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1, zero); + // SAFETY: the entrypoint establishes AES and NEON for these valid state registers. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1); + } } let mut offset = 0usize; while offset.strict_add(BLOCK_SIZE) <= aad.len() { - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - vld1q_u8(aad.as_ptr().add(offset)), - zero, - ); + // SAFETY: the loop bound provides a complete readable block at `offset`; the entrypoint enables AES and NEON. + unsafe { + update_regs( + &mut s0, + &mut s1, + &mut s2, + &mut s3, + &mut s4, + &mut s5, + vld1q_u8(aad.as_ptr().add(offset)), + ); + } offset = offset.strict_add(BLOCK_SIZE); } if offset < aad.len() { let mut pad = [0u8; BLOCK_SIZE]; pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pad), zero); + // SAFETY: `pad` is a complete initialized block, and the entrypoint enables AES and NEON. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pad)); + } } let ct_len = buffer.len(); let ptr = buffer.as_mut_ptr(); @@ -252,71 +306,77 @@ pub(super) unsafe fn decrypt_fused( let four_blocks = BLOCK_SIZE.strict_mul(4); let two_blocks = BLOCK_SIZE.strict_mul(2); while offset.strict_add(four_blocks) <= len { - core::arch::asm!("prfm pldl1keep, [{ptr}]", ptr = in(reg) ptr.add(offset.strict_add(192)), options(nostack, preserves_flags)); - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let ci_a = vld1q_u8(ptr.add(offset)); - let xi_a = veorq_u8(ci_a, z_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a, zero); - vst1q_u8(ptr.add(offset), xi_a); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let ci_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); - let xi_b = veorq_u8(ci_b, z_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b, zero); - vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), xi_b); - let z_c = keystream_regs(s1, s2, s3, s4, s5); - let ci_c = vld1q_u8(ptr.add(offset.strict_add(two_blocks))); - let xi_c = veorq_u8(ci_c, z_c); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c, zero); - vst1q_u8(ptr.add(offset.strict_add(two_blocks)), xi_c); - let z_d = keystream_regs(s1, s2, s3, s4, s5); - let ci_d = vld1q_u8(ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE)))); - let xi_d = veorq_u8(ci_d, z_d); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d, zero); - vst1q_u8(ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE))), xi_d); + prefetch_read_l1(ptr.wrapping_add(offset.strict_add(192))); + // SAFETY: the loop bound places all four 16-byte lanes inside the exclusively borrowed buffer. The entrypoint + // enables AES and NEON, and each ciphertext lane is loaded before its plaintext overwrites the same bytes. + unsafe { + let z_a = keystream_regs(s1, s2, s3, s4, s5); + let ci_a = vld1q_u8(ptr.add(offset)); + let xi_a = veorq_u8(ci_a, z_a); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); + vst1q_u8(ptr.add(offset), xi_a); + let z_b = keystream_regs(s1, s2, s3, s4, s5); + let ci_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); + let xi_b = veorq_u8(ci_b, z_b); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); + vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), xi_b); + let z_c = keystream_regs(s1, s2, s3, s4, s5); + let ci_c = vld1q_u8(ptr.add(offset.strict_add(two_blocks))); + let xi_c = veorq_u8(ci_c, z_c); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c); + vst1q_u8(ptr.add(offset.strict_add(two_blocks)), xi_c); + let z_d = keystream_regs(s1, s2, s3, s4, s5); + let ci_d = vld1q_u8(ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE)))); + let xi_d = veorq_u8(ci_d, z_d); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d); + vst1q_u8(ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE))), xi_d); + } offset = offset.strict_add(four_blocks); } if offset.strict_add(two_blocks) <= len { - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let ci_a = vld1q_u8(ptr.add(offset)); - let xi_a = veorq_u8(ci_a, z_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a, zero); - vst1q_u8(ptr.add(offset), xi_a); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let ci_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); - let xi_b = veorq_u8(ci_b, z_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b, zero); - vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), xi_b); + // SAFETY: the branch bound places both lanes inside the exclusively borrowed buffer; AES and NEON are enabled. + unsafe { + let z_a = keystream_regs(s1, s2, s3, s4, s5); + let ci_a = vld1q_u8(ptr.add(offset)); + let xi_a = veorq_u8(ci_a, z_a); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); + vst1q_u8(ptr.add(offset), xi_a); + let z_b = keystream_regs(s1, s2, s3, s4, s5); + let ci_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); + let xi_b = veorq_u8(ci_b, z_b); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); + vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), xi_b); + } offset = offset.strict_add(two_blocks); } if offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream_regs(s1, s2, s3, s4, s5); - let ci = vld1q_u8(ptr.add(offset)); - let xi = veorq_u8(ci, z); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi, zero); - vst1q_u8(ptr.add(offset), xi); + // SAFETY: the branch bound provides one complete writable lane in the buffer; AES and NEON are enabled. + unsafe { + let z = keystream_regs(s1, s2, s3, s4, s5); + let ci = vld1q_u8(ptr.add(offset)); + let xi = veorq_u8(ci, z); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); + vst1q_u8(ptr.add(offset), xi); + } offset = offset.strict_add(BLOCK_SIZE); } if offset < len { - let z = keystream_regs(s1, s2, s3, s4, s5); + // SAFETY: the entrypoint enables NEON for the valid state registers. + let z = unsafe { keystream_regs(s1, s2, s3, s4, s5) }; let tail_len = len.strict_sub(offset); let mut pad = [0u8; BLOCK_SIZE]; pad[..tail_len].copy_from_slice(&buffer[offset..]); let mut z_bytes = [0u8; BLOCK_SIZE]; - store(z, &mut z_bytes); + // SAFETY: `z_bytes` is a complete writable block, and the entrypoint enables NEON. + unsafe { store(z, &mut z_bytes) }; let mut pt_pad = [0u8; BLOCK_SIZE]; for i in 0..tail_len { pt_pad[i] = pad[i] ^ z_bytes[i]; } - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - load(&pt_pad), - zero, - ); + // SAFETY: `pt_pad` is a complete initialized block, and the entrypoint enables AES and NEON. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pt_pad)); + } buffer[offset..].copy_from_slice(&pt_pad[..tail_len]); } let ad_bits = (aad.len() as u64).strict_mul(8); @@ -324,12 +384,17 @@ pub(super) unsafe fn decrypt_fused( let mut len_bytes = [0u8; BLOCK_SIZE]; len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); len_bytes[8..].copy_from_slice(&ct_bits.to_le_bytes()); - let t = veorq_u8(s3, load(&len_bytes)); + // SAFETY: `len_bytes` is a complete initialized block, and the entrypoint enables NEON. + let t = unsafe { veorq_u8(s3, load(&len_bytes)) }; for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t, zero); + // SAFETY: the entrypoint establishes AES and NEON for these valid state registers. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t); + } } let tag_vec = veorq_u8(veorq_u8(veorq_u8(s0, s1), veorq_u8(s2, s3)), veorq_u8(s4, s5)); let mut tag = [0u8; TAG_SIZE]; - store(tag_vec, &mut tag); + // SAFETY: `tag` is a complete writable block, and the entrypoint enables NEON. + unsafe { store(tag_vec, &mut tag) }; tag } diff --git a/src/aead/aegis256/powerpc64_ppc.rs b/src/aead/aegis256/powerpc64_ppc.rs index d6d2c1f0..ab3da5af 100644 --- a/src/aead/aegis256/powerpc64_ppc.rs +++ b/src/aead/aegis256/powerpc64_ppc.rs @@ -25,8 +25,8 @@ fn store_be(v: i64x2, out: &mut [u8; 16]) { let arr = v.to_array(); #[cfg(target_endian = "little")] { - let hi = (arr[1] as u64).to_be_bytes(); - let lo = (arr[0] as u64).to_be_bytes(); + let hi = u64::from_ne_bytes(arr[1].to_ne_bytes()).to_be_bytes(); + let lo = u64::from_ne_bytes(arr[0].to_ne_bytes()).to_be_bytes(); out[0..8].copy_from_slice(&hi); out[8..16].copy_from_slice(&lo); } @@ -43,18 +43,19 @@ fn store_be(v: i64x2, out: &mut [u8; 16]) { /// Caller must ensure POWER8 vector crypto support is available. unsafe fn aes_round(block: i64x2, round_key: i64x2) -> i64x2 { let out: i64x2; - asm!( - "vcipher {out}, {block}, {rk}", - out = lateout(vreg) out, - block = in(vreg) block, - rk = in(vreg) round_key, - options(nomem, nostack), - ); + // SAFETY: the caller guarantees POWER8 vector crypto support; the instruction only reads its register operands. + unsafe { + asm!( + "vcipher {out}, {block}, {rk}", + out = lateout(vreg) out, + block = in(vreg) block, + rk = in(vreg) round_key, + options(nomem, nostack), + ) + }; out } -// ── Register-based helpers ────────────────────────────────────────────── - #[inline(always)] fn xor_vec(a: i64x2, b: i64x2) -> i64x2 { let aa = a.to_array(); @@ -69,27 +70,44 @@ fn and_vec(a: i64x2, b: i64x2) -> i64x2 { i64x2::from_array([aa[0] & ba[0], aa[1] & ba[1]]) } -#[inline(always)] -/// # Safety -/// -/// Caller must ensure POWER8 vector crypto support is available and all state -/// registers belong to a valid AEGIS-256 state. -unsafe fn update_regs( - s0: &mut i64x2, - s1: &mut i64x2, - s2: &mut i64x2, - s3: &mut i64x2, - s4: &mut i64x2, - s5: &mut i64x2, - m: i64x2, -) { - let tmp = *s5; - *s5 = aes_round(*s4, *s5); - *s4 = aes_round(*s3, *s4); - *s3 = aes_round(*s2, *s3); - *s2 = aes_round(*s1, *s2); - *s1 = aes_round(*s0, *s1); - *s0 = xor_vec(aes_round(tmp, *s0), m); +type StateMut<'a> = ( + &'a mut i64x2, + &'a mut i64x2, + &'a mut i64x2, + &'a mut i64x2, + &'a mut i64x2, + &'a mut i64x2, +); + +#[derive(Clone, Copy)] +struct Power8 { + _private: (), +} + +impl Power8 { + /// Creates a POWER8 vector-crypto capability token. + /// + /// # Safety + /// + /// The current CPU must support POWER8 vector crypto for the token's entire lifetime. + #[inline] + unsafe fn new() -> Self { + Self { _private: () } + } + + #[inline(always)] + fn update(self, (s0, s1, s2, s3, s4, s5): StateMut<'_>, message: i64x2) { + let old_s5 = *s5; + // SAFETY: this module constructs `Power8` only through `new`, whose caller guarantees POWER8 vector crypto. + unsafe { + *s5 = aes_round(*s4, *s5); + *s4 = aes_round(*s3, *s4); + *s3 = aes_round(*s2, *s3); + *s2 = aes_round(*s1, *s2); + *s1 = aes_round(*s0, *s1); + *s0 = xor_vec(aes_round(old_s5, *s0), message); + } + } } #[inline(always)] @@ -97,8 +115,6 @@ fn keystream_regs(s1: i64x2, s2: i64x2, s3: i64x2, s4: i64x2, s5: i64x2) -> i64x xor_vec(xor_vec(s1, s4), xor_vec(s5, and_vec(s2, s3))) } -// ── Fused encrypt/decrypt ─────────────────────────────────────────────── - #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] /// # Safety /// @@ -109,6 +125,8 @@ pub(super) unsafe fn encrypt_fused( aad: &[u8], buffer: &mut [u8], ) -> [u8; TAG_SIZE] { + // SAFETY: this function's caller guarantees POWER8 vector crypto support. + let power8 = unsafe { Power8::new() }; let (kh0, kh1) = super::split_halves(key); let (nh0, nh1) = super::split_halves(nonce); let k0 = load_be(kh0); @@ -122,118 +140,63 @@ pub(super) unsafe fn encrypt_fused( let (mut s0, mut s1, mut s2, mut s3, mut s4, mut s5) = (k0_xor_n0, k1_xor_n1, c1, c0, xor_vec(k0, c0), xor_vec(k1, c1)); for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0_xor_n0); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1_xor_n1); } - let mut offset = 0usize; - while offset.strict_add(BLOCK_SIZE) <= aad.len() { - let mut tmp = [0u8; 16]; - tmp.copy_from_slice(&aad[offset..offset.strict_add(BLOCK_SIZE)]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load_be(&tmp)); - offset = offset.strict_add(BLOCK_SIZE); + + let (aad_blocks, aad_tail) = aad.as_chunks::(); + for block in aad_blocks { + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load_be(block)); } - if offset < aad.len() { + if !aad_tail.is_empty() { let mut pad = [0u8; BLOCK_SIZE]; - pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load_be(&pad)); + let (pad_tail, _) = pad.split_at_mut(aad_tail.len()); + pad_tail.copy_from_slice(aad_tail); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load_be(&pad)); } + let msg_len = buffer.len(); - let len = buffer.len(); - offset = 0; - let four_blocks = BLOCK_SIZE.strict_mul(4); - let two_blocks = BLOCK_SIZE.strict_mul(2); - while offset.strict_add(four_blocks) <= len { - // SAFETY: pointer arithmetic for dcbt prefetch; offset + 256 may exceed - // the buffer but dcbt is a hint and never faults on POWER. - asm!("dcbt 0, {ptr}", ptr = in(reg) buffer.as_ptr().add(offset.strict_add(256)), options(nostack)); - // block a - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let mut tmp_a = [0u8; 16]; - tmp_a.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi_a = load_be(&tmp_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - store_be(xor_vec(xi_a, z_a), &mut tmp_a); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_a); - // block b - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let off_b = offset.strict_add(BLOCK_SIZE); - let mut tmp_b = [0u8; 16]; - tmp_b.copy_from_slice(&buffer[off_b..off_b.strict_add(BLOCK_SIZE)]); - let xi_b = load_be(&tmp_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - store_be(xor_vec(xi_b, z_b), &mut tmp_b); - buffer[off_b..off_b.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_b); - // block c - let z_c = keystream_regs(s1, s2, s3, s4, s5); - let off_c = offset.strict_add(two_blocks); - let mut tmp_c = [0u8; 16]; - tmp_c.copy_from_slice(&buffer[off_c..off_c.strict_add(BLOCK_SIZE)]); - let xi_c = load_be(&tmp_c); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c); - store_be(xor_vec(xi_c, z_c), &mut tmp_c); - buffer[off_c..off_c.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_c); - // block d - let z_d = keystream_regs(s1, s2, s3, s4, s5); - let off_d = offset.strict_add(two_blocks.strict_add(BLOCK_SIZE)); - let mut tmp_d = [0u8; 16]; - tmp_d.copy_from_slice(&buffer[off_d..off_d.strict_add(BLOCK_SIZE)]); - let xi_d = load_be(&tmp_d); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d); - store_be(xor_vec(xi_d, z_d), &mut tmp_d); - buffer[off_d..off_d.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_d); - offset = offset.strict_add(four_blocks); - } - if offset.strict_add(two_blocks) <= len { - // block a - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let mut tmp_a = [0u8; 16]; - tmp_a.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi_a = load_be(&tmp_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - store_be(xor_vec(xi_a, z_a), &mut tmp_a); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_a); - // block b - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let off_b = offset.strict_add(BLOCK_SIZE); - let mut tmp_b = [0u8; 16]; - tmp_b.copy_from_slice(&buffer[off_b..off_b.strict_add(BLOCK_SIZE)]); - let xi_b = load_be(&tmp_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - store_be(xor_vec(xi_b, z_b), &mut tmp_b); - buffer[off_b..off_b.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_b); - offset = offset.strict_add(two_blocks); + let (quads, remainder) = buffer.as_chunks_mut::<64>(); + for quad in quads { + let (blocks, _) = quad.as_chunks_mut::(); + for block in blocks { + let stream = keystream_regs(s1, s2, s3, s4, s5); + let plaintext = load_be(block); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store_be(xor_vec(plaintext, stream), block); + } } - if offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream_regs(s1, s2, s3, s4, s5); - let mut tmp = [0u8; 16]; - tmp.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi = load_be(&tmp); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); - store_be(xor_vec(xi, z), &mut tmp); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp); - offset = offset.strict_add(BLOCK_SIZE); + let (blocks, tail) = remainder.as_chunks_mut::(); + for block in blocks { + let stream = keystream_regs(s1, s2, s3, s4, s5); + let plaintext = load_be(block); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store_be(xor_vec(plaintext, stream), block); } - if offset < len { - let z = keystream_regs(s1, s2, s3, s4, s5); - let tail_len = len.strict_sub(offset); + if !tail.is_empty() { + let stream = keystream_regs(s1, s2, s3, s4, s5); let mut pad = [0u8; BLOCK_SIZE]; - pad[..tail_len].copy_from_slice(&buffer[offset..]); - let xi = load_be(&pad); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); - let mut ct_bytes = [0u8; BLOCK_SIZE]; - store_be(xor_vec(xi, z), &mut ct_bytes); - buffer[offset..].copy_from_slice(&ct_bytes[..tail_len]); + let (pad_tail, _) = pad.split_at_mut(tail.len()); + pad_tail.copy_from_slice(tail); + let plaintext = load_be(&pad); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + let mut ciphertext = [0u8; BLOCK_SIZE]; + store_be(xor_vec(plaintext, stream), &mut ciphertext); + let (ciphertext_tail, _) = ciphertext.split_at(tail.len()); + tail.copy_from_slice(ciphertext_tail); } + let ad_bits = (aad.len() as u64).strict_mul(8); let msg_bits = (msg_len as u64).strict_mul(8); let mut len_bytes = [0u8; BLOCK_SIZE]; - len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); - len_bytes[8..].copy_from_slice(&msg_bits.to_le_bytes()); + let (ad_len_bytes, msg_len_bytes) = len_bytes.split_at_mut(8); + ad_len_bytes.copy_from_slice(&ad_bits.to_le_bytes()); + msg_len_bytes.copy_from_slice(&msg_bits.to_le_bytes()); let t = xor_vec(s3, load_be(&len_bytes)); for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), t); } let tag_vec = xor_vec(xor_vec(xor_vec(s0, s1), xor_vec(s2, s3)), xor_vec(s4, s5)); let mut tag = [0u8; TAG_SIZE]; @@ -251,6 +214,8 @@ pub(super) unsafe fn decrypt_fused( aad: &[u8], buffer: &mut [u8], ) -> [u8; TAG_SIZE] { + // SAFETY: this function's caller guarantees POWER8 vector crypto support. + let power8 = unsafe { Power8::new() }; let (kh0, kh1) = super::split_halves(key); let (nh0, nh1) = super::split_halves(nonce); let k0 = load_be(kh0); @@ -264,121 +229,66 @@ pub(super) unsafe fn decrypt_fused( let (mut s0, mut s1, mut s2, mut s3, mut s4, mut s5) = (k0_xor_n0, k1_xor_n1, c1, c0, xor_vec(k0, c0), xor_vec(k1, c1)); for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0_xor_n0); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1_xor_n1); } - let mut offset = 0usize; - while offset.strict_add(BLOCK_SIZE) <= aad.len() { - let mut tmp = [0u8; 16]; - tmp.copy_from_slice(&aad[offset..offset.strict_add(BLOCK_SIZE)]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load_be(&tmp)); - offset = offset.strict_add(BLOCK_SIZE); + + let (aad_blocks, aad_tail) = aad.as_chunks::(); + for block in aad_blocks { + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load_be(block)); } - if offset < aad.len() { + if !aad_tail.is_empty() { let mut pad = [0u8; BLOCK_SIZE]; - pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load_be(&pad)); + let (pad_tail, _) = pad.split_at_mut(aad_tail.len()); + pad_tail.copy_from_slice(aad_tail); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load_be(&pad)); } + let ct_len = buffer.len(); - let len = buffer.len(); - offset = 0; - let four_blocks = BLOCK_SIZE.strict_mul(4); - let two_blocks = BLOCK_SIZE.strict_mul(2); - while offset.strict_add(four_blocks) <= len { - // SAFETY: pointer arithmetic for dcbt prefetch; offset + 256 may exceed - // the buffer but dcbt is a hint and never faults on POWER. - asm!("dcbt 0, {ptr}", ptr = in(reg) buffer.as_ptr().add(offset.strict_add(256)), options(nostack)); - // block a - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let mut tmp_a = [0u8; 16]; - tmp_a.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi_a = xor_vec(load_be(&tmp_a), z_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - store_be(xi_a, &mut tmp_a); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_a); - // block b - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let off_b = offset.strict_add(BLOCK_SIZE); - let mut tmp_b = [0u8; 16]; - tmp_b.copy_from_slice(&buffer[off_b..off_b.strict_add(BLOCK_SIZE)]); - let xi_b = xor_vec(load_be(&tmp_b), z_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - store_be(xi_b, &mut tmp_b); - buffer[off_b..off_b.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_b); - // block c - let z_c = keystream_regs(s1, s2, s3, s4, s5); - let off_c = offset.strict_add(two_blocks); - let mut tmp_c = [0u8; 16]; - tmp_c.copy_from_slice(&buffer[off_c..off_c.strict_add(BLOCK_SIZE)]); - let xi_c = xor_vec(load_be(&tmp_c), z_c); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c); - store_be(xi_c, &mut tmp_c); - buffer[off_c..off_c.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_c); - // block d - let z_d = keystream_regs(s1, s2, s3, s4, s5); - let off_d = offset.strict_add(two_blocks.strict_add(BLOCK_SIZE)); - let mut tmp_d = [0u8; 16]; - tmp_d.copy_from_slice(&buffer[off_d..off_d.strict_add(BLOCK_SIZE)]); - let xi_d = xor_vec(load_be(&tmp_d), z_d); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d); - store_be(xi_d, &mut tmp_d); - buffer[off_d..off_d.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_d); - offset = offset.strict_add(four_blocks); - } - if offset.strict_add(two_blocks) <= len { - // block a - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let mut tmp_a = [0u8; 16]; - tmp_a.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi_a = xor_vec(load_be(&tmp_a), z_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - store_be(xi_a, &mut tmp_a); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_a); - // block b - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let off_b = offset.strict_add(BLOCK_SIZE); - let mut tmp_b = [0u8; 16]; - tmp_b.copy_from_slice(&buffer[off_b..off_b.strict_add(BLOCK_SIZE)]); - let xi_b = xor_vec(load_be(&tmp_b), z_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - store_be(xi_b, &mut tmp_b); - buffer[off_b..off_b.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_b); - offset = offset.strict_add(two_blocks); + let (quads, remainder) = buffer.as_chunks_mut::<64>(); + for quad in quads { + let (blocks, _) = quad.as_chunks_mut::(); + for block in blocks { + let stream = keystream_regs(s1, s2, s3, s4, s5); + let plaintext = xor_vec(load_be(block), stream); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store_be(plaintext, block); + } } - if offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream_regs(s1, s2, s3, s4, s5); - let mut tmp = [0u8; 16]; - tmp.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi = xor_vec(load_be(&tmp), z); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); - store_be(xi, &mut tmp); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp); - offset = offset.strict_add(BLOCK_SIZE); + let (blocks, tail) = remainder.as_chunks_mut::(); + for block in blocks { + let stream = keystream_regs(s1, s2, s3, s4, s5); + let plaintext = xor_vec(load_be(block), stream); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store_be(plaintext, block); } - if offset < len { - let z = keystream_regs(s1, s2, s3, s4, s5); - let tail_len = len.strict_sub(offset); - let mut pad = [0u8; BLOCK_SIZE]; - pad[..tail_len].copy_from_slice(&buffer[offset..]); - let mut z_bytes = [0u8; BLOCK_SIZE]; - store_be(z, &mut z_bytes); - let mut pt_pad = [0u8; BLOCK_SIZE]; - for i in 0..tail_len { - pt_pad[i] = pad[i] ^ z_bytes[i]; + if !tail.is_empty() { + let stream = keystream_regs(s1, s2, s3, s4, s5); + let mut stream_bytes = [0u8; BLOCK_SIZE]; + store_be(stream, &mut stream_bytes); + let mut plaintext = [0u8; BLOCK_SIZE]; + for ((out, ciphertext), mask) in plaintext.iter_mut().zip(tail.iter()).zip(stream_bytes) { + *out = *ciphertext ^ mask; } - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load_be(&pt_pad)); - buffer[offset..].copy_from_slice(&pt_pad[..tail_len]); + power8.update( + (&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), + load_be(&plaintext), + ); + let (plaintext_tail, _) = plaintext.split_at(tail.len()); + tail.copy_from_slice(plaintext_tail); } + let ad_bits = (aad.len() as u64).strict_mul(8); let ct_bits = (ct_len as u64).strict_mul(8); let mut len_bytes = [0u8; BLOCK_SIZE]; - len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); - len_bytes[8..].copy_from_slice(&ct_bits.to_le_bytes()); + let (ad_len_bytes, ct_len_bytes) = len_bytes.split_at_mut(8); + ad_len_bytes.copy_from_slice(&ad_bits.to_le_bytes()); + ct_len_bytes.copy_from_slice(&ct_bits.to_le_bytes()); let t = xor_vec(s3, load_be(&len_bytes)); for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), t); } let tag_vec = xor_vec(xor_vec(xor_vec(s0, s1), xor_vec(s2, s3)), xor_vec(s4, s5)); let mut tag = [0u8; TAG_SIZE]; diff --git a/src/aead/aegis256/riscv64_vperm.rs b/src/aead/aegis256/riscv64_vperm.rs deleted file mode 100644 index 4a7e4289..00000000 --- a/src/aead/aegis256/riscv64_vperm.rs +++ /dev/null @@ -1,368 +0,0 @@ -use core::arch::asm; - -use super::{BLOCK_SIZE, Block, C0, C1, KEY_SIZE, NONCE_SIZE, TAG_SIZE, and_block, split_halves, xor_block}; -use crate::aead::aes_round::{ - AES_AFFINE, MC_ROT1, MC_ROT2, VPERM_INV_HI, VPERM_INV_LO, VPERM_IPT_HI, VPERM_IPT_LO, VPERM_SBOT, VPERM_SBOU, - VPERM_SR, XTIME_REDUCE, -}; - -type State = [Block; 6]; - -/// Precomputed Hamburg vperm table block — packed contiguously for -/// offset-based vector loads in the asm block. -#[repr(C, align(16))] -struct VpermTables { - ipt_lo: [u8; 16], // offset 0 - ipt_hi: [u8; 16], // offset 16 - inv_lo: [u8; 16], // offset 32 - inv_hi: [u8; 16], // offset 48 - sbou: [u8; 16], // offset 64 - sbot: [u8; 16], // offset 80 - sr_perm: [u8; 16], // offset 96 - mc_rot1: [u8; 16], // offset 112 - mc_rot2: [u8; 16], // offset 128 - affine: [u8; 16], // offset 144 - xtime: [u8; 16], // offset 160 -} - -impl VpermTables { - #[inline(always)] - fn load() -> Self { - Self { - ipt_lo: VPERM_IPT_LO, - ipt_hi: VPERM_IPT_HI, - inv_lo: VPERM_INV_LO, - inv_hi: VPERM_INV_HI, - sbou: VPERM_SBOU, - sbot: VPERM_SBOT, - sr_perm: VPERM_SR, - mc_rot1: MC_ROT1, - mc_rot2: MC_ROT2, - affine: [AES_AFFINE; 16], - xtime: [XTIME_REDUCE; 16], - } - } -} - -/// Single AES round via Hamburg vperm on RISC-V V: SubBytes + ShiftRows + -/// MixColumns + AddRoundKey. Uses `vrgather.vv` for all S-box nibble lookups. -/// -/// # Safety -/// Requires the RISC-V V extension. -#[target_feature(enable = "v")] -#[inline] -unsafe fn aes_round(block: &Block, round_key: &Block, tables: &VpermTables) -> Block { - let mut out = [0u8; BLOCK_SIZE]; - - // SAFETY: Caller guarantees RISC-V V extension is available. - // The asm block loads the state, tables, and round key from memory, - // performs all computation in vector registers, and stores the result. - // All vrgather indices are masked to 0-15 before lookup — no secret- - // dependent memory access. - unsafe { - asm!( - // ── Setup ────────────────────────────────────────────────────── - "vsetivli zero, 16, e8, m1, ta, ma", - - // Load tables from the VpermTables struct (contiguous, 16B each). - "vle8.v v2, ({tbl})", // IPT_LO (offset 0) - "addi {tmp}, {tbl}, 16", - "vle8.v v3, ({tmp})", // IPT_HI (offset 16) - "addi {tmp}, {tbl}, 32", - "vle8.v v4, ({tmp})", // INV_LO (offset 32) - "addi {tmp}, {tbl}, 48", - "vle8.v v5, ({tmp})", // INV_HI (offset 48) - "addi {tmp}, {tbl}, 64", - "vle8.v v6, ({tmp})", // SBOU (offset 64) - "addi {tmp}, {tbl}, 80", - "vle8.v v7, ({tmp})", // SBOT (offset 80) - "addi {tmp}, {tbl}, 96", - "vle8.v v8, ({tmp})", // SR_PERM (offset 96) - "addi {tmp}, {tbl}, 112", - "vle8.v v9, ({tmp})", // MC_ROT1 (offset 112) - "addi {tmp}, {tbl}, 128", - "vle8.v v10, ({tmp})", // MC_ROT2 (offset 128) - "addi {tmp}, {tbl}, 144", - "vle8.v v11, ({tmp})", // 0x63 (offset 144) - "addi {tmp}, {tbl}, 160", - "vle8.v v12, ({tmp})", // 0x1B (offset 160) - - // Load state and round key. - "vle8.v v0, ({state})", - "vle8.v v1, ({rk})", - - // ── Phase 1: Nibble extraction ───────────────────────────────── - "vand.vi v14, v0, 15", // lo_nib = state & 0x0F - "vsrl.vi v15, v0, 4", // hi_nib = state >> 4 - - // ── Phase 2: Input transform (AES → tower field) ────────────── - "vrgather.vv v16, v2, v14", // ipt_l = IPT_LO[lo_nib] - "vrgather.vv v17, v3, v15", // ipt_h = IPT_HI[hi_nib] - "vxor.vv v14, v16, v17", // x = ipt_l ^ ipt_h - - // ── Phase 3: Re-extract nibbles of transformed value ─────────── - "vand.vi v15, v14, 15", // t_lo = x & 0x0F - "vsrl.vi v16, v14, 4", // t_hi = x >> 4 - - // ── Phase 4: GF(2^4) inverse ────────────────────────────────── - "vrgather.vv v17, v5, v15", // ak = INV_HI[t_lo] - "vxor.vv v18, v16, v15", // j = t_hi ^ t_lo - "vrgather.vv v19, v4, v16", // inv_i = INV_LO[t_hi] - "vxor.vv v20, v19, v17", // iak = inv_i ^ ak - "vrgather.vv v21, v4, v18", // inv_j = INV_LO[j] - "vxor.vv v22, v21, v17", // jak = inv_j ^ ak - - // vperm_z(INV_LO, iak): zero where bit 7 set - "vand.vi v23, v20, 15", // iak & 0x0F - "vrgather.vv v24, v4, v23", // INV_LO[iak & 0x0F] - "vsra.vi v25, v20, 7", // 0xFF where bit 7 set - "vxor.vi v26, v25, -1", // ~mask - "vand.vv v24, v24, v26", // zero masked positions - "vxor.vv v14, v24, v18", // io = inv_iak ^ j - - // vperm_z(INV_LO, jak): zero where bit 7 set - "vand.vi v23, v22, 15", - "vrgather.vv v24, v4, v23", - "vsra.vi v25, v22, 7", - "vxor.vi v26, v25, -1", - "vand.vv v24, v24, v26", - "vxor.vv v15, v24, v16", // jo = inv_jak ^ t_hi - - // ── Phase 5: Output transform (SubBytes) ────────────────────── - // vperm_z(SBOU, io) - "vand.vi v23, v14, 15", - "vrgather.vv v24, v6, v23", - "vsra.vi v25, v14, 7", - "vxor.vi v26, v25, -1", - "vand.vv v16, v24, v26", // su - - // vperm_z(SBOT, jo) - "vand.vi v23, v15, 15", - "vrgather.vv v24, v7, v23", - "vsra.vi v25, v15, 7", - "vxor.vi v26, v25, -1", - "vand.vv v17, v24, v26", // st - - // sb = su ^ st ^ 0x63 - "vxor.vv v14, v16, v17", - "vxor.vv v14, v14, v11", - - // ── ShiftRows ───────────────────────────────────────────────── - "vrgather.vv v15, v14, v8", // sr = sb permuted - - // ── MixColumns (xtime decomposition) ────────────────────────── - "vrgather.vv v16, v15, v9", // rot1 = column-rotate-by-1 - "vxor.vv v17, v15, v16", // pair = sr ^ rot1 - "vsll.vi v18, v17, 1", // pair << 1 - "vsra.vi v19, v17, 7", // bit-7 mask for xtime reduction - "vand.vv v19, v19, v12", // mask & 0x1B - "vxor.vv v18, v18, v19", // xt = xtime(pair) - "vrgather.vv v19, v17, v10", // rot2_pair = column-rotate-by-2 - "vxor.vv v20, v17, v19", // col_sum = pair ^ rot2_pair - "vxor.vv v14, v15, v20", // sr ^ col_sum - "vxor.vv v14, v14, v18", // mc = sr ^ col_sum ^ xt - - // ── AddRoundKey ─────────────────────────────────────────────── - "vxor.vv v0, v14, v1", - - // ── Store result ────────────────────────────────────────────── - "vse8.v v0, ({out})", - - state = in(reg) block.as_ptr(), - rk = in(reg) round_key.as_ptr(), - tbl = in(reg) tables as *const VpermTables as *const u8, - out = in(reg) out.as_mut_ptr(), - tmp = out(reg) _, - options(nostack), - ); - } - out -} - -// ── AEGIS-256 state operations ────────────────────────────────────────── - -#[target_feature(enable = "v")] -#[inline] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -unsafe fn update(s: &mut State, m: &Block, tables: &VpermTables) { - // SAFETY: caller guarantees the RISC-V V extension is available and the - // state blocks are valid local buffers for the Hamburg round function. - unsafe { - let tmp = s[5]; - s[5] = aes_round(&s[4], &s[5], tables); - s[4] = aes_round(&s[3], &s[4], tables); - s[3] = aes_round(&s[2], &s[3], tables); - s[2] = aes_round(&s[1], &s[2], tables); - s[1] = aes_round(&s[0], &s[1], tables); - s[0] = xor_block(&aes_round(&tmp, &s[0], tables), m); - } -} - -#[inline(always)] -fn keystream(s: &State) -> Block { - xor_block(&xor_block(&s[1], &s[4]), &xor_block(&s[5], &and_block(&s[2], &s[3]))) -} - -// ── Fused encrypt/decrypt ───────────────────────────────────────────── - -#[target_feature(enable = "v")] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -pub(super) unsafe fn encrypt_fused( - key: &[u8; KEY_SIZE], - nonce: &[u8; NONCE_SIZE], - aad: &[u8], - buffer: &mut [u8], -) -> [u8; TAG_SIZE] { - // SAFETY: caller guarantees the RISC-V V extension is available for the - // lifetime of this fused operation and all slices are valid Rust references. - unsafe { - let tables = VpermTables::load(); - let (kh0, kh1) = split_halves(key); - let (nh0, nh1) = split_halves(nonce); - let k0_xor_n0 = xor_block(kh0, nh0); - let k1_xor_n1 = xor_block(kh1, nh1); - let mut s: State = [k0_xor_n0, k1_xor_n1, C1, C0, xor_block(kh0, &C0), xor_block(kh1, &C1)]; - - for _ in 0..4 { - update(&mut s, kh0, &tables); - update(&mut s, kh1, &tables); - update(&mut s, &k0_xor_n0, &tables); - update(&mut s, &k1_xor_n1, &tables); - } - - let mut offset = 0usize; - while offset.strict_add(BLOCK_SIZE) <= aad.len() { - let mut tmp = [0u8; BLOCK_SIZE]; - tmp.copy_from_slice(&aad[offset..offset.strict_add(BLOCK_SIZE)]); - update(&mut s, &tmp, &tables); - offset = offset.strict_add(BLOCK_SIZE); - } - if offset < aad.len() { - let mut pad = [0u8; BLOCK_SIZE]; - pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update(&mut s, &pad, &tables); - } - - let msg_len = buffer.len(); - let len = buffer.len(); - offset = 0; - while offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream(&s); - let mut xi = [0u8; BLOCK_SIZE]; - xi.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - update(&mut s, &xi, &tables); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&xor_block(&xi, &z)); - offset = offset.strict_add(BLOCK_SIZE); - } - if offset < len { - let z = keystream(&s); - let tail_len = len.strict_sub(offset); - let mut pad = [0u8; BLOCK_SIZE]; - pad[..tail_len].copy_from_slice(&buffer[offset..]); - update(&mut s, &pad, &tables); - let ct = xor_block(&pad, &z); - buffer[offset..].copy_from_slice(&ct[..tail_len]); - } - - let ad_bits = (aad.len() as u64).strict_mul(8); - let msg_bits = (msg_len as u64).strict_mul(8); - let mut len_bytes = [0u8; BLOCK_SIZE]; - len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); - len_bytes[8..].copy_from_slice(&msg_bits.to_le_bytes()); - let t = xor_block(&s[3], &len_bytes); - for _ in 0..7 { - update(&mut s, &t, &tables); - } - - xor_block( - &xor_block(&xor_block(&s[0], &s[1]), &xor_block(&s[2], &s[3])), - &xor_block(&s[4], &s[5]), - ) - } -} - -#[target_feature(enable = "v")] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -pub(super) unsafe fn decrypt_fused( - key: &[u8; KEY_SIZE], - nonce: &[u8; NONCE_SIZE], - aad: &[u8], - buffer: &mut [u8], -) -> [u8; TAG_SIZE] { - // SAFETY: caller guarantees the RISC-V V extension is available for the - // lifetime of this fused operation and all slices are valid Rust references. - unsafe { - let tables = VpermTables::load(); - let (kh0, kh1) = split_halves(key); - let (nh0, nh1) = split_halves(nonce); - let k0_xor_n0 = xor_block(kh0, nh0); - let k1_xor_n1 = xor_block(kh1, nh1); - let mut s: State = [k0_xor_n0, k1_xor_n1, C1, C0, xor_block(kh0, &C0), xor_block(kh1, &C1)]; - - for _ in 0..4 { - update(&mut s, kh0, &tables); - update(&mut s, kh1, &tables); - update(&mut s, &k0_xor_n0, &tables); - update(&mut s, &k1_xor_n1, &tables); - } - - let mut offset = 0usize; - while offset.strict_add(BLOCK_SIZE) <= aad.len() { - let mut tmp = [0u8; BLOCK_SIZE]; - tmp.copy_from_slice(&aad[offset..offset.strict_add(BLOCK_SIZE)]); - update(&mut s, &tmp, &tables); - offset = offset.strict_add(BLOCK_SIZE); - } - if offset < aad.len() { - let mut pad = [0u8; BLOCK_SIZE]; - pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update(&mut s, &pad, &tables); - } - - let ct_len = buffer.len(); - let len = buffer.len(); - offset = 0; - while offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream(&s); - let mut ci = [0u8; BLOCK_SIZE]; - ci.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi = xor_block(&ci, &z); - update(&mut s, &xi, &tables); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&xi); - offset = offset.strict_add(BLOCK_SIZE); - } - if offset < len { - let z = keystream(&s); - let tail_len = len.strict_sub(offset); - let mut pad = [0u8; BLOCK_SIZE]; - pad[..tail_len].copy_from_slice(&buffer[offset..]); - let mut pt_pad = [0u8; BLOCK_SIZE]; - for i in 0..tail_len { - pt_pad[i] = pad[i] ^ z[i]; - } - update(&mut s, &pt_pad, &tables); - buffer[offset..].copy_from_slice(&pt_pad[..tail_len]); - } - - let ad_bits = (aad.len() as u64).strict_mul(8); - let ct_bits = (ct_len as u64).strict_mul(8); - let mut len_bytes = [0u8; BLOCK_SIZE]; - len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); - len_bytes[8..].copy_from_slice(&ct_bits.to_le_bytes()); - let t = xor_block(&s[3], &len_bytes); - for _ in 0..7 { - update(&mut s, &t, &tables); - } - - xor_block( - &xor_block(&xor_block(&s[0], &s[1]), &xor_block(&s[2], &s[3])), - &xor_block(&s[4], &s[5]), - ) - } -} diff --git a/src/aead/aegis256/s390x_vperm.rs b/src/aead/aegis256/s390x_vperm.rs index 11c7fe59..f22d1530 100644 --- a/src/aead/aegis256/s390x_vperm.rs +++ b/src/aead/aegis256/s390x_vperm.rs @@ -48,7 +48,7 @@ fn and_vec(a: i64x2, b: i64x2) -> i64x2 { /// Broadcast a byte to all 16 positions of a vector. #[inline(always)] fn splat_byte(b: u8) -> i64x2 { - let w = u64::from_ne_bytes([b; 8]) as i64; + let w = i64::from_ne_bytes([b; 8]); i64x2::from_array([w, w]) } @@ -297,38 +297,31 @@ impl VpermTables { // ── AEGIS-256 state operations ────────────────────────────────────────── #[target_feature(enable = "vector")] -#[allow(clippy::too_many_arguments)] #[inline] /// # Safety /// /// Caller must ensure the s390x vector facility is available and all state /// registers belong to a valid AEGIS-256 state. -unsafe fn update_regs( - s0: &mut i64x2, - s1: &mut i64x2, - s2: &mut i64x2, - s3: &mut i64x2, - s4: &mut i64x2, - s5: &mut i64x2, - m: i64x2, - tables: &VpermTables, -) { +unsafe fn update_regs(state: &mut [i64x2; 6], m: i64x2, tables: &VpermTables) { // SAFETY: caller guarantees the s390x vector facility is available for // this helper and all state registers are valid local values. unsafe { - let tmp = *s5; - *s5 = aes_round(*s4, *s5, tables); - *s4 = aes_round(*s3, *s4, tables); - *s3 = aes_round(*s2, *s3, tables); - *s2 = aes_round(*s1, *s2, tables); - *s1 = aes_round(*s0, *s1, tables); - *s0 = xor_vec(aes_round(tmp, *s0, tables), m); + let tmp = state[5]; + state[5] = aes_round(state[4], state[5], tables); + state[4] = aes_round(state[3], state[4], tables); + state[3] = aes_round(state[2], state[3], tables); + state[2] = aes_round(state[1], state[2], tables); + state[1] = aes_round(state[0], state[1], tables); + state[0] = xor_vec(aes_round(tmp, state[0], tables), m); } } #[inline(always)] -fn keystream_regs(s1: i64x2, s2: i64x2, s3: i64x2, s4: i64x2, s5: i64x2) -> i64x2 { - xor_vec(xor_vec(s1, s4), xor_vec(s5, and_vec(s2, s3))) +fn keystream_regs(state: &[i64x2; 6]) -> i64x2 { + xor_vec( + xor_vec(state[1], state[4]), + xor_vec(state[5], and_vec(state[2], state[3])), + ) } // ── Fused encrypt/decrypt ───────────────────────────────────────────── @@ -357,64 +350,45 @@ pub(super) unsafe fn encrypt_fused( let c1 = load_be(&C1); let k0_xor_n0 = xor_vec(k0, n0); let k1_xor_n1 = xor_vec(k1, n1); - let (mut s0, mut s1, mut s2, mut s3, mut s4, mut s5) = - (k0_xor_n0, k1_xor_n1, c1, c0, xor_vec(k0, c0), xor_vec(k1, c1)); + let mut state = [k0_xor_n0, k1_xor_n1, c1, c0, xor_vec(k0, c0), xor_vec(k1, c1)]; for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0, &tables); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1, &tables); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0, &tables); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1, &tables); + update_regs(&mut state, k0, &tables); + update_regs(&mut state, k1, &tables); + update_regs(&mut state, k0_xor_n0, &tables); + update_regs(&mut state, k1_xor_n1, &tables); } let mut offset = 0usize; while offset.strict_add(BLOCK_SIZE) <= aad.len() { let mut tmp = [0u8; 16]; tmp.copy_from_slice(&aad[offset..offset.strict_add(BLOCK_SIZE)]); - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - load_be(&tmp), - &tables, - ); + update_regs(&mut state, load_be(&tmp), &tables); offset = offset.strict_add(BLOCK_SIZE); } if offset < aad.len() { let mut pad = [0u8; BLOCK_SIZE]; pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - load_be(&pad), - &tables, - ); + update_regs(&mut state, load_be(&pad), &tables); } let msg_len = buffer.len(); let len = buffer.len(); offset = 0; while offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream_regs(s1, s2, s3, s4, s5); + let z = keystream_regs(&state); let mut tmp = [0u8; 16]; tmp.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); let xi = load_be(&tmp); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi, &tables); + update_regs(&mut state, xi, &tables); store_be(xor_vec(xi, z), &mut tmp); buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp); offset = offset.strict_add(BLOCK_SIZE); } if offset < len { - let z = keystream_regs(s1, s2, s3, s4, s5); + let z = keystream_regs(&state); let tail_len = len.strict_sub(offset); let mut pad = [0u8; BLOCK_SIZE]; pad[..tail_len].copy_from_slice(&buffer[offset..]); let xi = load_be(&pad); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi, &tables); + update_regs(&mut state, xi, &tables); let mut ct_bytes = [0u8; BLOCK_SIZE]; store_be(xor_vec(xi, z), &mut ct_bytes); buffer[offset..].copy_from_slice(&ct_bytes[..tail_len]); @@ -424,11 +398,14 @@ pub(super) unsafe fn encrypt_fused( let mut len_bytes = [0u8; BLOCK_SIZE]; len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); len_bytes[8..].copy_from_slice(&msg_bits.to_le_bytes()); - let t = xor_vec(s3, load_be(&len_bytes)); + let t = xor_vec(state[3], load_be(&len_bytes)); for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t, &tables); + update_regs(&mut state, t, &tables); } - let tag_vec = xor_vec(xor_vec(xor_vec(s0, s1), xor_vec(s2, s3)), xor_vec(s4, s5)); + let tag_vec = xor_vec( + xor_vec(xor_vec(state[0], state[1]), xor_vec(state[2], state[3])), + xor_vec(state[4], state[5]), + ); let mut tag = [0u8; TAG_SIZE]; store_be(tag_vec, &mut tag); tag @@ -459,59 +436,40 @@ pub(super) unsafe fn decrypt_fused( let c1 = load_be(&C1); let k0_xor_n0 = xor_vec(k0, n0); let k1_xor_n1 = xor_vec(k1, n1); - let (mut s0, mut s1, mut s2, mut s3, mut s4, mut s5) = - (k0_xor_n0, k1_xor_n1, c1, c0, xor_vec(k0, c0), xor_vec(k1, c1)); + let mut state = [k0_xor_n0, k1_xor_n1, c1, c0, xor_vec(k0, c0), xor_vec(k1, c1)]; for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0, &tables); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1, &tables); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0, &tables); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1, &tables); + update_regs(&mut state, k0, &tables); + update_regs(&mut state, k1, &tables); + update_regs(&mut state, k0_xor_n0, &tables); + update_regs(&mut state, k1_xor_n1, &tables); } let mut offset = 0usize; while offset.strict_add(BLOCK_SIZE) <= aad.len() { let mut tmp = [0u8; 16]; tmp.copy_from_slice(&aad[offset..offset.strict_add(BLOCK_SIZE)]); - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - load_be(&tmp), - &tables, - ); + update_regs(&mut state, load_be(&tmp), &tables); offset = offset.strict_add(BLOCK_SIZE); } if offset < aad.len() { let mut pad = [0u8; BLOCK_SIZE]; pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - load_be(&pad), - &tables, - ); + update_regs(&mut state, load_be(&pad), &tables); } let ct_len = buffer.len(); let len = buffer.len(); offset = 0; while offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream_regs(s1, s2, s3, s4, s5); + let z = keystream_regs(&state); let mut tmp = [0u8; 16]; tmp.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); let xi = xor_vec(load_be(&tmp), z); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi, &tables); + update_regs(&mut state, xi, &tables); store_be(xi, &mut tmp); buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp); offset = offset.strict_add(BLOCK_SIZE); } if offset < len { - let z = keystream_regs(s1, s2, s3, s4, s5); + let z = keystream_regs(&state); let tail_len = len.strict_sub(offset); let mut pad = [0u8; BLOCK_SIZE]; pad[..tail_len].copy_from_slice(&buffer[offset..]); @@ -521,16 +479,7 @@ pub(super) unsafe fn decrypt_fused( for i in 0..tail_len { pt_pad[i] = pad[i] ^ z_bytes[i]; } - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - load_be(&pt_pad), - &tables, - ); + update_regs(&mut state, load_be(&pt_pad), &tables); buffer[offset..].copy_from_slice(&pt_pad[..tail_len]); } let ad_bits = (aad.len() as u64).strict_mul(8); @@ -538,11 +487,14 @@ pub(super) unsafe fn decrypt_fused( let mut len_bytes = [0u8; BLOCK_SIZE]; len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); len_bytes[8..].copy_from_slice(&ct_bits.to_le_bytes()); - let t = xor_vec(s3, load_be(&len_bytes)); + let t = xor_vec(state[3], load_be(&len_bytes)); for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t, &tables); + update_regs(&mut state, t, &tables); } - let tag_vec = xor_vec(xor_vec(xor_vec(s0, s1), xor_vec(s2, s3)), xor_vec(s4, s5)); + let tag_vec = xor_vec( + xor_vec(xor_vec(state[0], state[1]), xor_vec(state[2], state[3])), + xor_vec(state[4], state[5]), + ); let mut tag = [0u8; TAG_SIZE]; store_be(tag_vec, &mut tag); tag diff --git a/src/aead/aegis256/x86_64_ni.rs b/src/aead/aegis256/x86_64_ni.rs index 3a8c3a5b..32813c71 100644 --- a/src/aead/aegis256/x86_64_ni.rs +++ b/src/aead/aegis256/x86_64_ni.rs @@ -3,66 +3,62 @@ use core::arch::x86_64::*; use super::{BLOCK_SIZE, C0, C1, KEY_SIZE, NONCE_SIZE, TAG_SIZE}; #[inline] -/// # Safety -/// -/// `bytes` must refer to a valid 16-byte block. -unsafe fn load(bytes: &[u8; BLOCK_SIZE]) -> __m128i { - _mm_loadu_si128(bytes.as_ptr().cast()) +fn load(bytes: &[u8; BLOCK_SIZE]) -> __m128i { + // SAFETY: SSE2 is an x86_64 baseline feature; `bytes` provides 16 readable bytes, and the load is unaligned. + unsafe { _mm_loadu_si128(bytes.as_ptr().cast()) } } #[inline] -/// # Safety -/// -/// `out` must refer to a valid writable 16-byte block. -unsafe fn store(v: __m128i, out: &mut [u8; BLOCK_SIZE]) { - _mm_storeu_si128(out.as_mut_ptr().cast(), v); +fn store(value: __m128i, out: &mut [u8; BLOCK_SIZE]) { + // SAFETY: SSE2 is an x86_64 baseline feature; `out` provides 16 writable bytes, and the store is unaligned. + unsafe { _mm_storeu_si128(out.as_mut_ptr().cast(), value) }; } -// ── Register-based helpers ────────────────────────────────────────────── -// -// All 6 AES rounds in an AEGIS-256 update read from the OLD state and are -// mutually independent. Keeping state in 6 local `__m128i` values (rather -// than indexing an array through `&mut State`) lets the register allocator -// pin them to XMM registers across loop iterations, eliminating store-to- -// load round-trips that serialize the OOO pipeline. +type StateMut<'a> = ( + &'a mut __m128i, + &'a mut __m128i, + &'a mut __m128i, + &'a mut __m128i, + &'a mut __m128i, + &'a mut __m128i, +); -#[inline] -/// # Safety -/// -/// Caller must ensure AES-NI is available and all register arguments come -/// from a valid AEGIS-256 state. -unsafe fn update_regs( - s0: &mut __m128i, - s1: &mut __m128i, - s2: &mut __m128i, - s3: &mut __m128i, - s4: &mut __m128i, - s5: &mut __m128i, - m: __m128i, -) { - let tmp = *s5; - *s5 = _mm_aesenc_si128(*s4, *s5); - *s4 = _mm_aesenc_si128(*s3, *s4); - *s3 = _mm_aesenc_si128(*s2, *s3); - *s2 = _mm_aesenc_si128(*s1, *s2); - *s1 = _mm_aesenc_si128(*s0, *s1); - *s0 = _mm_xor_si128(_mm_aesenc_si128(tmp, *s0), m); +#[derive(Clone, Copy)] +struct AesNi { + _private: (), } -#[inline] -/// # Safety -/// -/// The provided registers must come from a valid AEGIS-256 state on an -/// AES-NI capable CPU. -unsafe fn keystream_regs(s1: __m128i, s2: __m128i, s3: __m128i, s4: __m128i, s5: __m128i) -> __m128i { - _mm_xor_si128(_mm_xor_si128(s1, s4), _mm_xor_si128(s5, _mm_and_si128(s2, s3))) +impl AesNi { + /// Creates an AES-NI capability token. + /// + /// # Safety + /// + /// The current CPU must support AES-NI and AVX for the token's entire lifetime. + #[inline] + unsafe fn new() -> Self { + Self { _private: () } + } + + #[inline(always)] + fn update(self, (s0, s1, s2, s3, s4, s5): StateMut<'_>, message: __m128i) { + let old_s5 = *s5; + // SAFETY: this module constructs `AesNi` only through `new`, whose caller guarantees AES-NI and AVX support. + unsafe { + *s5 = _mm_aesenc_si128(*s4, *s5); + *s4 = _mm_aesenc_si128(*s3, *s4); + *s3 = _mm_aesenc_si128(*s2, *s3); + *s2 = _mm_aesenc_si128(*s1, *s2); + *s1 = _mm_aesenc_si128(*s0, *s1); + *s0 = _mm_xor_si128(_mm_aesenc_si128(old_s5, *s0), message); + } + } } -// ── Fused encrypt/decrypt ─────────────────────────────────────────────── -// -// Single `#[target_feature]` entry points that keep state in XMM registers -// from init through finalize, eliminating ~15 cycles of stack spills that -// occur when init/aad/encrypt/finalize are separate function calls. +#[inline] +fn keystream(s1: __m128i, s2: __m128i, s3: __m128i, s4: __m128i, s5: __m128i) -> __m128i { + // SAFETY: SSE2 is part of the x86_64 baseline. + unsafe { _mm_xor_si128(_mm_xor_si128(s1, s4), _mm_xor_si128(s5, _mm_and_si128(s2, s3))) } +} #[target_feature(enable = "aes,avx")] /// # Safety @@ -74,7 +70,8 @@ pub(super) unsafe fn encrypt_fused( aad: &[u8], buffer: &mut [u8], ) -> [u8; TAG_SIZE] { - // ── init ── + // SAFETY: this function's caller guarantees AES-NI and AVX support. + let aesni = unsafe { AesNi::new() }; let (kh0, kh1) = super::split_halves(key); let (nh0, nh1) = super::split_halves(nonce); let k0 = load(kh0); @@ -94,94 +91,63 @@ pub(super) unsafe fn encrypt_fused( _mm_xor_si128(k1, c1), ); for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0_xor_n0); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1_xor_n1); } - // ── aad ── - let mut offset = 0usize; - while offset.strict_add(BLOCK_SIZE) <= aad.len() { - let block = _mm_loadu_si128(aad.as_ptr().add(offset).cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, block); - offset = offset.strict_add(BLOCK_SIZE); + let (aad_blocks, aad_tail) = aad.as_chunks::(); + for block in aad_blocks { + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load(block)); } - if offset < aad.len() { + if !aad_tail.is_empty() { let mut pad = [0u8; BLOCK_SIZE]; - pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pad)); + let (pad_tail, _) = pad.split_at_mut(aad_tail.len()); + pad_tail.copy_from_slice(aad_tail); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load(&pad)); } - // ── encrypt ── - // - // Counted loop with raw pointer advancement: eliminates per-iteration - // overflow-check branches (strict_add) from the hot path. The loop count - // is pre-validated via integer division, and all intra-block offsets are - // compile-time constants applied through ptr::add inside this unsafe block. let msg_len = buffer.len(); - let ptr = buffer.as_mut_ptr(); - let n_quads = msg_len / 64; - let mut p = ptr; - for _ in 0..n_quads { - _mm_prefetch(p.add(256).cast::(), _MM_HINT_T0); - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let xi_a = _mm_loadu_si128(p.cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - _mm_storeu_si128(p.cast(), _mm_xor_si128(xi_a, z_a)); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let xi_b = _mm_loadu_si128(p.add(16).cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - _mm_storeu_si128(p.add(16).cast(), _mm_xor_si128(xi_b, z_b)); - let z_c = keystream_regs(s1, s2, s3, s4, s5); - let xi_c = _mm_loadu_si128(p.add(32).cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c); - _mm_storeu_si128(p.add(32).cast(), _mm_xor_si128(xi_c, z_c)); - let z_d = keystream_regs(s1, s2, s3, s4, s5); - let xi_d = _mm_loadu_si128(p.add(48).cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d); - _mm_storeu_si128(p.add(48).cast(), _mm_xor_si128(xi_d, z_d)); - p = p.add(64); - } - let mut remaining = msg_len.strict_sub(n_quads.strict_mul(64)); - if remaining >= 32 { - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let xi_a = _mm_loadu_si128(p.cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - _mm_storeu_si128(p.cast(), _mm_xor_si128(xi_a, z_a)); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let xi_b = _mm_loadu_si128(p.add(16).cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - _mm_storeu_si128(p.add(16).cast(), _mm_xor_si128(xi_b, z_b)); - p = p.add(32); - remaining = remaining.strict_sub(32); + let (quads, remainder) = buffer.as_chunks_mut::<64>(); + for quad in quads { + let (blocks, _) = quad.as_chunks_mut::(); + for block in blocks { + let stream = keystream(s1, s2, s3, s4, s5); + let plaintext = load(block); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store(_mm_xor_si128(plaintext, stream), block); + } } - if remaining >= 16 { - let z = keystream_regs(s1, s2, s3, s4, s5); - let xi = _mm_loadu_si128(p.cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); - _mm_storeu_si128(p.cast(), _mm_xor_si128(xi, z)); - remaining = remaining.strict_sub(16); + let (blocks, tail) = remainder.as_chunks_mut::(); + for block in blocks { + let stream = keystream(s1, s2, s3, s4, s5); + let plaintext = load(block); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store(_mm_xor_si128(plaintext, stream), block); } - if remaining > 0 { - let z = keystream_regs(s1, s2, s3, s4, s5); - let tail_off = msg_len.strict_sub(remaining); + if !tail.is_empty() { + let stream = keystream(s1, s2, s3, s4, s5); let mut pad = [0u8; BLOCK_SIZE]; - pad[..remaining].copy_from_slice(&buffer[tail_off..]); - let xi = load(&pad); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); - let mut ct_bytes = [0u8; BLOCK_SIZE]; - store(_mm_xor_si128(xi, z), &mut ct_bytes); - buffer[tail_off..].copy_from_slice(&ct_bytes[..remaining]); + let (pad_tail, _) = pad.split_at_mut(tail.len()); + pad_tail.copy_from_slice(tail); + let plaintext = load(&pad); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + let mut ciphertext = [0u8; BLOCK_SIZE]; + store(_mm_xor_si128(plaintext, stream), &mut ciphertext); + let (ciphertext_tail, _) = ciphertext.split_at(tail.len()); + tail.copy_from_slice(ciphertext_tail); } - // ── finalize ── let ad_bits = (aad.len() as u64).strict_mul(8); let msg_bits = (msg_len as u64).strict_mul(8); - let len_block = _mm_set_epi64x(msg_bits as i64, ad_bits as i64); + let len_block = _mm_set_epi64x( + i64::from_ne_bytes(msg_bits.to_ne_bytes()), + i64::from_ne_bytes(ad_bits.to_ne_bytes()), + ); let t = _mm_xor_si128(s3, len_block); for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), t); } let tag_vec = _mm_xor_si128( _mm_xor_si128(_mm_xor_si128(s0, s1), _mm_xor_si128(s2, s3)), @@ -202,7 +168,8 @@ pub(super) unsafe fn decrypt_fused( aad: &[u8], buffer: &mut [u8], ) -> [u8; TAG_SIZE] { - // ── init ── + // SAFETY: this function's caller guarantees AES-NI and AVX support. + let aesni = unsafe { AesNi::new() }; let (kh0, kh1) = super::split_halves(key); let (nh0, nh1) = super::split_halves(nonce); let k0 = load(kh0); @@ -222,102 +189,63 @@ pub(super) unsafe fn decrypt_fused( _mm_xor_si128(k1, c1), ); for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0_xor_n0); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1_xor_n1); } - // ── aad ── - let mut offset = 0usize; - while offset.strict_add(BLOCK_SIZE) <= aad.len() { - let block = _mm_loadu_si128(aad.as_ptr().add(offset).cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, block); - offset = offset.strict_add(BLOCK_SIZE); + let (aad_blocks, aad_tail) = aad.as_chunks::(); + for block in aad_blocks { + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load(block)); } - if offset < aad.len() { + if !aad_tail.is_empty() { let mut pad = [0u8; BLOCK_SIZE]; - pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pad)); + let (pad_tail, _) = pad.split_at_mut(aad_tail.len()); + pad_tail.copy_from_slice(aad_tail); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load(&pad)); } - // ── decrypt ── - // - // Same counted-loop strategy as encrypt: raw pointer advancement with - // compile-time-constant offsets, zero overflow checks in the hot path. let ct_len = buffer.len(); - let ptr = buffer.as_mut_ptr(); - let n_quads = ct_len / 64; - let mut p = ptr; - for _ in 0..n_quads { - _mm_prefetch(p.add(256).cast::(), _MM_HINT_T0); - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let ci_a = _mm_loadu_si128(p.cast()); - let xi_a = _mm_xor_si128(ci_a, z_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - _mm_storeu_si128(p.cast(), xi_a); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let ci_b = _mm_loadu_si128(p.add(16).cast()); - let xi_b = _mm_xor_si128(ci_b, z_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - _mm_storeu_si128(p.add(16).cast(), xi_b); - let z_c = keystream_regs(s1, s2, s3, s4, s5); - let ci_c = _mm_loadu_si128(p.add(32).cast()); - let xi_c = _mm_xor_si128(ci_c, z_c); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c); - _mm_storeu_si128(p.add(32).cast(), xi_c); - let z_d = keystream_regs(s1, s2, s3, s4, s5); - let ci_d = _mm_loadu_si128(p.add(48).cast()); - let xi_d = _mm_xor_si128(ci_d, z_d); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d); - _mm_storeu_si128(p.add(48).cast(), xi_d); - p = p.add(64); - } - let mut remaining = ct_len.strict_sub(n_quads.strict_mul(64)); - if remaining >= 32 { - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let ci_a = _mm_loadu_si128(p.cast()); - let xi_a = _mm_xor_si128(ci_a, z_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - _mm_storeu_si128(p.cast(), xi_a); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let ci_b = _mm_loadu_si128(p.add(16).cast()); - let xi_b = _mm_xor_si128(ci_b, z_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - _mm_storeu_si128(p.add(16).cast(), xi_b); - p = p.add(32); - remaining = remaining.strict_sub(32); + let (quads, remainder) = buffer.as_chunks_mut::<64>(); + for quad in quads { + let (blocks, _) = quad.as_chunks_mut::(); + for block in blocks { + let stream = keystream(s1, s2, s3, s4, s5); + let plaintext = _mm_xor_si128(load(block), stream); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store(plaintext, block); + } } - if remaining >= 16 { - let z = keystream_regs(s1, s2, s3, s4, s5); - let ci = _mm_loadu_si128(p.cast()); - let xi = _mm_xor_si128(ci, z); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); - _mm_storeu_si128(p.cast(), xi); - remaining = remaining.strict_sub(16); + let (blocks, tail) = remainder.as_chunks_mut::(); + for block in blocks { + let stream = keystream(s1, s2, s3, s4, s5); + let plaintext = _mm_xor_si128(load(block), stream); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store(plaintext, block); } - if remaining > 0 { - let z = keystream_regs(s1, s2, s3, s4, s5); - let tail_off = ct_len.strict_sub(remaining); - let mut pad = [0u8; BLOCK_SIZE]; - pad[..remaining].copy_from_slice(&buffer[tail_off..]); - let mut z_bytes = [0u8; BLOCK_SIZE]; - store(z, &mut z_bytes); - let mut pt_pad = [0u8; BLOCK_SIZE]; - for i in 0..remaining { - pt_pad[i] = pad[i] ^ z_bytes[i]; + if !tail.is_empty() { + let stream = keystream(s1, s2, s3, s4, s5); + let mut stream_bytes = [0u8; BLOCK_SIZE]; + store(stream, &mut stream_bytes); + let mut plaintext = [0u8; BLOCK_SIZE]; + for ((out, ciphertext), mask) in plaintext.iter_mut().zip(tail.iter()).zip(stream_bytes) { + *out = *ciphertext ^ mask; } - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pt_pad)); - buffer[tail_off..].copy_from_slice(&pt_pad[..remaining]); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load(&plaintext)); + let (plaintext_tail, _) = plaintext.split_at(tail.len()); + tail.copy_from_slice(plaintext_tail); } - // ── finalize ── let ad_bits = (aad.len() as u64).strict_mul(8); let ct_bits = (ct_len as u64).strict_mul(8); - let len_block = _mm_set_epi64x(ct_bits as i64, ad_bits as i64); + let len_block = _mm_set_epi64x( + i64::from_ne_bytes(ct_bits.to_ne_bytes()), + i64::from_ne_bytes(ad_bits.to_ne_bytes()), + ); let t = _mm_xor_si128(s3, len_block); for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), t); } let tag_vec = _mm_xor_si128( _mm_xor_si128(_mm_xor_si128(s0, s1), _mm_xor_si128(s2, s3)), diff --git a/src/aead/aes.rs b/src/aead/aes.rs index e77afab3..2b5d572d 100644 --- a/src/aead/aes.rs +++ b/src/aead/aes.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! Portable table-free AES block-cipher core with hardware dispatch. //! //! This module provides AES-128 and AES-256 key expansion and single-block @@ -40,31 +38,23 @@ pub(crate) const EXPANDED_KEY_WORDS_128: usize = 4 * (ROUNDS_128 + 1); // 44 #[path = "aes/aarch64_ce.rs"] mod ce; #[cfg(target_arch = "s390x")] -#[allow(unsafe_code)] #[path = "aes/s390x_km.rs"] mod km; #[cfg(target_arch = "x86_64")] #[path = "aes/x86_64_ni.rs"] mod ni; #[cfg(target_arch = "powerpc64")] -#[allow(unsafe_code)] #[path = "aes/powerpc64_ppc.rs"] mod ppc; #[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] #[path = "aes/riscv64_aes.rs"] mod rv_aes; #[cfg(any(target_arch = "riscv64", test))] #[path = "aes/riscv64_fixslice_aes.rs"] mod rv_fixslice_aes; #[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] #[path = "aes/riscv64_scalar_aes.rs"] mod rv_scalar_aes; -#[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] -#[path = "aes/riscv64_vperm_aes.rs"] -mod rv_vperm_aes; #[cfg(all( target_arch = "x86_64", target_os = "linux", @@ -113,7 +103,7 @@ pub(crate) struct Aes256EncKey { } enum KeyInner { - #[allow(dead_code)] + #[cfg(not(target_arch = "riscv64"))] PortableRoundKeys([u32; EXPANDED_KEY_WORDS]), #[cfg(target_arch = "x86_64")] X86AesNi(ni::NiRoundKeys), @@ -124,24 +114,21 @@ enum KeyInner { #[cfg(target_arch = "powerpc64")] Power8Crypto(ppc::PpcRoundKeys), #[cfg(target_arch = "riscv64")] - Riscv64ScalarCrypto(rv_scalar_aes::RvScalarRoundKeys), - #[cfg(target_arch = "riscv64")] - Riscv64VectorCrypto(rv_aes::RvRoundKeys), - /// Hamburg vperm via `vrgather.vv` with the table-free portable key schedule. + ScalarCrypto(rv_scalar_aes::RvScalarRoundKeys), #[cfg(target_arch = "riscv64")] - #[allow(dead_code)] // V-only AES is kept for GCM-SIV and explicit diagnostic paths; GCM does not select it yet. - Riscv64Vperm([u32; EXPANDED_KEY_WORDS]), + VectorCrypto(rv_aes::RvRoundKeys), /// Four-block table-free fixslice fallback for scalar RV64 without AES extensions. #[cfg(all(target_arch = "riscv64", feature = "alloc"))] - Riscv64Fixslice(alloc::boxed::Box), + Fixslice(alloc::boxed::Box), /// No-alloc RV64 builds keep the larger fixslice key schedule inline. #[cfg(all(target_arch = "riscv64", not(feature = "alloc")))] - Riscv64Fixslice(rv_fixslice_aes::RvFixsliceRoundKeys), + Fixslice(rv_fixslice_aes::RvFixsliceRoundKeys), } impl Drop for Aes256EncKey { fn drop(&mut self) { match &mut self.inner { + #[cfg(not(target_arch = "riscv64"))] KeyInner::PortableRoundKeys(rk) => { // SAFETY: [u32; 60] is layout-compatible with [u8; 240]. crate::traits::ct::zeroize(unsafe { @@ -161,23 +148,15 @@ impl Drop for Aes256EncKey { ppc_rk.zeroize(); } #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64ScalarCrypto(rv_rk) => { + KeyInner::ScalarCrypto(rv_rk) => { rv_rk.zeroize(); } #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64VectorCrypto(rv_rk) => { + KeyInner::VectorCrypto(rv_rk) => { rv_rk.zeroize(); } #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64Vperm(rk) => { - // SAFETY: the expanded key is a contiguous `[u32; 60]`, so viewing it as - // a mutable byte slice for zeroization is valid for its exact size. - crate::traits::ct::zeroize(unsafe { - core::slice::from_raw_parts_mut(rk.as_mut_ptr().cast::(), EXPANDED_KEY_WORDS.strict_mul(4)) - }); - } - #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64Fixslice(rk) => { + KeyInner::Fixslice(rk) => { rk.zeroize(); } } @@ -195,7 +174,7 @@ pub(crate) struct Aes128EncKey { } enum Key128Inner { - #[allow(dead_code)] + #[cfg(not(target_arch = "riscv64"))] PortableRoundKeys([u32; EXPANDED_KEY_WORDS_128]), #[cfg(target_arch = "x86_64")] X86AesNi(ni::Ni128RoundKeys), @@ -206,24 +185,21 @@ enum Key128Inner { #[cfg(target_arch = "powerpc64")] Power8Crypto(ppc::Ppc128RoundKeys), #[cfg(target_arch = "riscv64")] - Riscv64ScalarCrypto(rv_scalar_aes::RvScalar128RoundKeys), + ScalarCrypto(rv_scalar_aes::RvScalar128RoundKeys), #[cfg(target_arch = "riscv64")] - Riscv64VectorCrypto(rv_aes::Rv128RoundKeys), - /// Hamburg vperm via `vrgather.vv` with the table-free portable key schedule. - #[cfg(target_arch = "riscv64")] - #[allow(dead_code)] // V-only AES is kept for GCM-SIV and explicit diagnostic paths; GCM does not select it yet. - Riscv64Vperm([u32; EXPANDED_KEY_WORDS_128]), + VectorCrypto(rv_aes::Rv128RoundKeys), /// Four-block table-free fixslice fallback for scalar RV64 without AES extensions. #[cfg(all(target_arch = "riscv64", feature = "alloc"))] - Riscv64Fixslice(alloc::boxed::Box), + Fixslice(alloc::boxed::Box), /// No-alloc RV64 builds keep the larger fixslice key schedule inline. #[cfg(all(target_arch = "riscv64", not(feature = "alloc")))] - Riscv64Fixslice(rv_fixslice_aes::RvFixslice128RoundKeys), + Fixslice(rv_fixslice_aes::RvFixslice128RoundKeys), } impl Drop for Aes128EncKey { fn drop(&mut self) { match &mut self.inner { + #[cfg(not(target_arch = "riscv64"))] Key128Inner::PortableRoundKeys(rk) => { // SAFETY: [u32; 44] is layout-compatible with [u8; 176]. crate::traits::ct::zeroize(unsafe { @@ -243,22 +219,15 @@ impl Drop for Aes128EncKey { ppc_rk.zeroize(); } #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64ScalarCrypto(rv_rk) => { + Key128Inner::ScalarCrypto(rv_rk) => { rv_rk.zeroize(); } #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64VectorCrypto(rv_rk) => { + Key128Inner::VectorCrypto(rv_rk) => { rv_rk.zeroize(); } #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64Vperm(rk) => { - // SAFETY: [u32; 44] is layout-compatible with [u8; 176]. - crate::traits::ct::zeroize(unsafe { - core::slice::from_raw_parts_mut(rk.as_mut_ptr().cast::(), EXPANDED_KEY_WORDS_128.strict_mul(4)) - }); - } - #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64Fixslice(rk) => { + Key128Inner::Fixslice(rk) => { rk.zeroize(); } } @@ -298,7 +267,7 @@ const fn gf256_mul(a: u8, b: u8) -> u8 { prod ^= (prod >> 9).wrapping_mul(0x11b << 1); prod ^= (prod >> 8).wrapping_mul(0x11b); - prod as u8 + prod.to_le_bytes()[0] } /// Square in GF(2^8). Equivalent to `gf256_mul(x, x)` but slightly cheaper. @@ -350,11 +319,8 @@ const fn sbox(x: u8) -> u8 { /// Apply SubBytes to a 32-bit word (four S-box applications). #[inline(always)] const fn sub_word(w: u32) -> u32 { - let b0 = sbox((w >> 24) as u8) as u32; - let b1 = sbox((w >> 16) as u8) as u32; - let b2 = sbox((w >> 8) as u8) as u32; - let b3 = sbox(w as u8) as u32; - (b0 << 24) | (b1 << 16) | (b2 << 8) | b3 + let [b0, b1, b2, b3] = w.to_be_bytes(); + u32::from_be_bytes([sbox(b0), sbox(b1), sbox(b2), sbox(b3)]) } /// Rotate a 32-bit word left by 8 bits. @@ -471,13 +437,13 @@ pub(crate) fn aes128_expand_key_portable(key: &[u8; KEY_SIZE_128]) -> [u32; EXPA #[cfg(all(target_arch = "riscv64", feature = "alloc"))] #[inline] fn riscv64_fixslice_key_inner(key: &[u8; KEY_SIZE]) -> KeyInner { - KeyInner::Riscv64Fixslice(alloc::boxed::Box::new(rv_fixslice_aes::RvFixsliceRoundKeys::new(key))) + KeyInner::Fixslice(alloc::boxed::Box::new(rv_fixslice_aes::RvFixsliceRoundKeys::new(key))) } #[cfg(all(target_arch = "riscv64", not(feature = "alloc")))] #[inline] fn riscv64_fixslice_key_inner(key: &[u8; KEY_SIZE]) -> KeyInner { - KeyInner::Riscv64Fixslice(rv_fixslice_aes::RvFixsliceRoundKeys::new(key)) + KeyInner::Fixslice(rv_fixslice_aes::RvFixsliceRoundKeys::new(key)) } /// Expand a 256-bit AES key into round keys. @@ -529,7 +495,7 @@ pub(crate) fn aes256_expand_key(key: &[u8; KEY_SIZE]) -> Aes256EncKey { let rv_keys = rv_aes::from_portable(&portable_rk); zeroize_expanded_key_words(&mut portable_rk); return Aes256EncKey { - inner: KeyInner::Riscv64VectorCrypto(rv_keys), + inner: KeyInner::VectorCrypto(rv_keys), }; } if crate::platform::caps().has(crate::platform::caps::riscv::ZKNE) { @@ -537,7 +503,7 @@ pub(crate) fn aes256_expand_key(key: &[u8; KEY_SIZE]) -> Aes256EncKey { let rv_keys = rv_scalar_aes::from_portable(&portable_rk); zeroize_expanded_key_words(&mut portable_rk); return Aes256EncKey { - inner: KeyInner::Riscv64ScalarCrypto(rv_keys), + inner: KeyInner::ScalarCrypto(rv_keys), }; } Aes256EncKey { @@ -553,7 +519,7 @@ pub(crate) fn aes256_expand_key(key: &[u8; KEY_SIZE]) -> Aes256EncKey { #[cfg(all(target_arch = "riscv64", feature = "alloc"))] #[inline] fn riscv64_fixslice_key_inner_128(key: &[u8; KEY_SIZE_128]) -> Key128Inner { - Key128Inner::Riscv64Fixslice(alloc::boxed::Box::new(rv_fixslice_aes::RvFixslice128RoundKeys::new( + Key128Inner::Fixslice(alloc::boxed::Box::new(rv_fixslice_aes::RvFixslice128RoundKeys::new( key, ))) } @@ -561,7 +527,7 @@ fn riscv64_fixslice_key_inner_128(key: &[u8; KEY_SIZE_128]) -> Key128Inner { #[cfg(all(target_arch = "riscv64", not(feature = "alloc")))] #[inline] fn riscv64_fixslice_key_inner_128(key: &[u8; KEY_SIZE_128]) -> Key128Inner { - Key128Inner::Riscv64Fixslice(rv_fixslice_aes::RvFixslice128RoundKeys::new(key)) + Key128Inner::Fixslice(rv_fixslice_aes::RvFixslice128RoundKeys::new(key)) } /// Expand a 128-bit AES key into round keys. @@ -613,7 +579,7 @@ pub(crate) fn aes128_expand_key(key: &[u8; KEY_SIZE_128]) -> Aes128EncKey { let rv_keys = rv_aes::from_portable_128(&portable_rk); zeroize_expanded_key_words_128(&mut portable_rk); return Aes128EncKey { - inner: Key128Inner::Riscv64VectorCrypto(rv_keys), + inner: Key128Inner::VectorCrypto(rv_keys), }; } if crate::platform::caps().has(crate::platform::caps::riscv::ZKNE) { @@ -621,7 +587,7 @@ pub(crate) fn aes128_expand_key(key: &[u8; KEY_SIZE_128]) -> Aes128EncKey { let rv_keys = rv_scalar_aes::from_portable_128(&portable_rk); zeroize_expanded_key_words_128(&mut portable_rk); return Aes128EncKey { - inner: Key128Inner::Riscv64ScalarCrypto(rv_keys), + inner: Key128Inner::ScalarCrypto(rv_keys), }; } Aes128EncKey { @@ -641,7 +607,7 @@ pub(crate) fn aes256_expand_key_riscv_vector(key: &[u8; KEY_SIZE]) -> Aes256EncK let rv_keys = rv_aes::from_portable(&portable_rk); zeroize_expanded_key_words(&mut portable_rk); Aes256EncKey { - inner: KeyInner::Riscv64VectorCrypto(rv_keys), + inner: KeyInner::VectorCrypto(rv_keys), } } @@ -652,15 +618,7 @@ pub(crate) fn aes256_expand_key_riscv_scalar(key: &[u8; KEY_SIZE]) -> Aes256EncK let rv_keys = rv_scalar_aes::from_portable(&portable_rk); zeroize_expanded_key_words(&mut portable_rk); Aes256EncKey { - inner: KeyInner::Riscv64ScalarCrypto(rv_keys), - } -} - -#[cfg(all(target_arch = "riscv64", feature = "aes-gcm-siv"))] -#[inline] -pub(crate) fn aes256_expand_key_riscv_vperm(key: &[u8; KEY_SIZE]) -> Aes256EncKey { - Aes256EncKey { - inner: KeyInner::Riscv64Vperm(aes256_expand_key_portable(key)), + inner: KeyInner::ScalarCrypto(rv_keys), } } @@ -680,7 +638,7 @@ pub(crate) fn aes128_expand_key_riscv_vector(key: &[u8; KEY_SIZE_128]) -> Aes128 let rv_keys = rv_aes::from_portable_128(&portable_rk); zeroize_expanded_key_words_128(&mut portable_rk); Aes128EncKey { - inner: Key128Inner::Riscv64VectorCrypto(rv_keys), + inner: Key128Inner::VectorCrypto(rv_keys), } } @@ -691,15 +649,7 @@ pub(crate) fn aes128_expand_key_riscv_scalar(key: &[u8; KEY_SIZE_128]) -> Aes128 let rv_keys = rv_scalar_aes::from_portable_128(&portable_rk); zeroize_expanded_key_words_128(&mut portable_rk); Aes128EncKey { - inner: Key128Inner::Riscv64ScalarCrypto(rv_keys), - } -} - -#[cfg(all(target_arch = "riscv64", feature = "aes-gcm-siv"))] -#[inline] -pub(crate) fn aes128_expand_key_riscv_vperm(key: &[u8; KEY_SIZE_128]) -> Aes128EncKey { - Aes128EncKey { - inner: Key128Inner::Riscv64Vperm(aes128_expand_key_portable(key)), + inner: Key128Inner::ScalarCrypto(rv_keys), } } @@ -800,11 +750,32 @@ unsafe fn aarch64_encrypt_blocks_inline(keys: &ce::CeRoundKeys, blocks: &mut [[u } } +#[cfg(all(target_arch = "aarch64", feature = "aes-gcm-siv"))] +#[inline] +fn gcmsiv_derive_keys_fallback(master_ek: &Aes256EncKey, nonce: &[u8; 12]) -> ([u8; 16], [u8; 32]) { + let mut blocks = [[0u8; BLOCK_SIZE]; 6]; + for (counter, block) in (0u32..6).zip(&mut blocks) { + block[..4].copy_from_slice(&counter.to_le_bytes()); + block[4..].copy_from_slice(nonce); + } + aes256_encrypt_blocks_ecb(master_ek, &mut blocks); + + let mut auth_key = [0u8; 16]; + let mut enc_key = [0u8; 32]; + auth_key[..8].copy_from_slice(&blocks[0][..8]); + auth_key[8..].copy_from_slice(&blocks[1][..8]); + enc_key[..8].copy_from_slice(&blocks[2][..8]); + enc_key[8..16].copy_from_slice(&blocks[3][..8]); + enc_key[16..24].copy_from_slice(&blocks[4][..8]); + enc_key[24..].copy_from_slice(&blocks[5][..8]); + crate::traits::ct::zeroize(blocks.as_flattened_mut()); + (auth_key, enc_key) +} + /// Derive AES-256-GCM-SIV per-message keys directly with AES-CE. /// /// # Safety -/// Caller must ensure AES-CE is available and `master_ek` is the AArch64 AES -/// backend variant. +/// Caller must ensure AES-CE is available. #[cfg(all(target_arch = "aarch64", feature = "aes-gcm-siv"))] #[target_feature(enable = "aes,neon")] #[inline] @@ -813,7 +784,7 @@ pub(super) unsafe fn aarch64_gcmsiv_derive_keys_inline( nonce: &[u8; 12], ) -> ([u8; 16], [u8; 32]) { let KeyInner::Aarch64Aes(ce_rk) = &master_ek.inner else { - unreachable!("AArch64 GCM-SIV KDF requires an AES-CE master key"); + return gcmsiv_derive_keys_fallback(master_ek, nonce); }; // SAFETY: direct AES-CE GCM-SIV KDF because: // 1. This function's caller must guarantee AES-CE availability. @@ -930,11 +901,30 @@ unsafe fn aarch64_encrypt_blocks_128_inline(keys: &ce::Ce128RoundKeys, blocks: & } } +#[cfg(all(target_arch = "aarch64", feature = "aes-gcm-siv"))] +#[inline] +fn gcmsiv_derive_keys_128_fallback(master_ek: &Aes128EncKey, nonce: &[u8; 12]) -> ([u8; 16], [u8; 16]) { + let mut blocks = [[0u8; BLOCK_SIZE]; 4]; + for (counter, block) in (0u32..4).zip(&mut blocks) { + block[..4].copy_from_slice(&counter.to_le_bytes()); + block[4..].copy_from_slice(nonce); + } + aes128_encrypt_blocks_ecb(master_ek, &mut blocks); + + let mut auth_key = [0u8; 16]; + let mut enc_key = [0u8; 16]; + auth_key[..8].copy_from_slice(&blocks[0][..8]); + auth_key[8..].copy_from_slice(&blocks[1][..8]); + enc_key[..8].copy_from_slice(&blocks[2][..8]); + enc_key[8..].copy_from_slice(&blocks[3][..8]); + crate::traits::ct::zeroize(blocks.as_flattened_mut()); + (auth_key, enc_key) +} + /// Derive AES-128-GCM-SIV per-message keys directly with AES-CE. /// /// # Safety -/// Caller must ensure AES-CE is available and `master_ek` is the AArch64 AES -/// backend variant. +/// Caller must ensure AES-CE is available. #[cfg(all(target_arch = "aarch64", feature = "aes-gcm-siv"))] #[target_feature(enable = "aes,neon")] #[inline] @@ -943,7 +933,7 @@ pub(super) unsafe fn aarch64_gcmsiv_derive_keys_128_inline( nonce: &[u8; 12], ) -> ([u8; 16], [u8; 16]) { let Key128Inner::Aarch64Aes(ce_rk) = &master_ek.inner else { - unreachable!("AArch64 GCM-SIV KDF requires an AES-CE master key"); + return gcmsiv_derive_keys_128_fallback(master_ek, nonce); }; // SAFETY: direct AES-CE GCM-SIV KDF because: // 1. This function's caller must guarantee AES-CE availability. @@ -1679,6 +1669,7 @@ pub(super) unsafe fn s390x_encrypt_blocks_128_inline(key: &km::Km128Key, blocks: #[inline] pub(crate) fn aes256_encrypt_block(ek: &Aes256EncKey, block: &mut [u8; BLOCK_SIZE]) { match &ek.inner { + #[cfg(not(target_arch = "riscv64"))] KeyInner::PortableRoundKeys(rk) => aes256_encrypt_block_portable(rk, block), #[cfg(target_arch = "x86_64")] KeyInner::X86AesNi(ni_rk) => { @@ -1701,27 +1692,21 @@ pub(crate) fn aes256_encrypt_block(ek: &Aes256EncKey, block: &mut [u8; BLOCK_SIZ unsafe { ppc::encrypt_block(ppc_rk, block) } } #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64ScalarCrypto(rv_rk) => { + KeyInner::ScalarCrypto(rv_rk) => { // SAFETY: RvScalar variant is only constructed after runtime detection confirms Zkne. unsafe { rv_scalar_aes::encrypt_block(rv_rk, block) } } #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64VectorCrypto(rv_rk) => { + KeyInner::VectorCrypto(rv_rk) => { // SAFETY: RvAes variant is only constructed after runtime detection confirms Zvkned. unsafe { rv_aes::encrypt_block(rv_rk, block) } } #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64Vperm(rk) => { - // SAFETY: RvVperm variant is only constructed after runtime detection confirms V extension. - unsafe { rv_vperm_aes::encrypt_block(rk, block) } - } - #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64Fixslice(rk) => rv_fixslice_aes::encrypt_block(rk, block), + KeyInner::Fixslice(rk) => rv_fixslice_aes::encrypt_block(rk, block), } } #[cfg(any(target_arch = "riscv64", test))] -#[allow(dead_code)] #[inline] pub(super) fn aes_enc_round_4_fixslice(blocks: &mut [[u8; BLOCK_SIZE]; 4], round_keys: &[[u8; BLOCK_SIZE]; 4]) { rv_fixslice_aes::cipher_round_4(blocks, round_keys); @@ -1736,6 +1721,7 @@ pub(super) fn aes_enc_round_4_fixslice(blocks: &mut [[u8; BLOCK_SIZE]; 4], round #[inline] pub(crate) fn aes128_encrypt_block(ek: &Aes128EncKey, block: &mut [u8; BLOCK_SIZE]) { match &ek.inner { + #[cfg(not(target_arch = "riscv64"))] Key128Inner::PortableRoundKeys(rk) => aes128_encrypt_block_portable(rk, block), #[cfg(target_arch = "x86_64")] Key128Inner::X86AesNi(ni_rk) => { @@ -1758,22 +1744,17 @@ pub(crate) fn aes128_encrypt_block(ek: &Aes128EncKey, block: &mut [u8; BLOCK_SIZ unsafe { ppc::encrypt_block_128(ppc_rk, block) } } #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64ScalarCrypto(rv_rk) => { - // SAFETY: Riscv64ScalarCrypto variant is only constructed after runtime detection confirms Zkne. + Key128Inner::ScalarCrypto(rv_rk) => { + // SAFETY: ScalarCrypto is only constructed after runtime detection confirms Zkne. unsafe { rv_scalar_aes::encrypt_block_128(rv_rk, block) } } #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64VectorCrypto(rv_rk) => { - // SAFETY: Riscv64VectorCrypto variant is only constructed after runtime detection confirms Zvkned. + Key128Inner::VectorCrypto(rv_rk) => { + // SAFETY: VectorCrypto is only constructed after runtime detection confirms Zvkned. unsafe { rv_aes::encrypt_block_128(rv_rk, block) } } #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64Vperm(rk) => { - // SAFETY: Riscv64Vperm variant is only constructed after runtime detection confirms V extension. - unsafe { rv_vperm_aes::encrypt_block_128(rk, block) } - } - #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64Fixslice(rk) => rv_fixslice_aes::encrypt_block_128(rk, block), + Key128Inner::Fixslice(rk) => rv_fixslice_aes::encrypt_block_128(rk, block), } } @@ -1783,17 +1764,6 @@ pub(crate) fn aes128_encrypt_block(ek: &Aes128EncKey, block: &mut [u8; BLOCK_SIZ /// instruction or the RV64 4-block kernels when available, otherwise calls /// the per-block dispatcher. Used by `riscv64` from the AES-128 CTR paths /// and by AES-128-GCM-SIV key derivation. -#[cfg_attr( - not(any( - target_arch = "aarch64", - target_arch = "powerpc64", - target_arch = "riscv64", - target_arch = "s390x", - feature = "aes-gcm-siv", - test - )), - allow(dead_code) -)] #[inline] pub(crate) fn aes128_encrypt_blocks_ecb(ek: &Aes128EncKey, blocks: &mut [[u8; BLOCK_SIZE]]) { #[cfg(target_arch = "aarch64")] @@ -1831,17 +1801,17 @@ pub(crate) fn aes128_encrypt_blocks_ecb(ek: &Aes128EncKey, blocks: &mut [[u8; BL return; } #[cfg(target_arch = "riscv64")] - if let Key128Inner::Riscv64VectorCrypto(rk) = &ek.inner { + if let Key128Inner::VectorCrypto(rk) = &ek.inner { let mut offset = 0usize; while offset.strict_add(4) <= blocks.len() { let batch_slice = &mut blocks[offset..offset.strict_add(4)]; debug_assert_eq!(batch_slice.len(), 4); // SAFETY: exact four-block RISC-V Zvkned AES-128 batch because: // 1. `batch_slice` is sliced to exactly four contiguous `[u8; 16]` elements. - // 2. `Riscv64VectorCrypto` is only constructed after runtime detection confirms Zvkned. + // 2. `VectorCrypto` is only constructed after runtime detection confirms Zvkned. // 3. The mutable borrow is scoped to this loop iteration. let batch: &mut [[u8; BLOCK_SIZE]; 4] = unsafe { &mut *batch_slice.as_mut_ptr().cast::<[[u8; BLOCK_SIZE]; 4]>() }; - // SAFETY: `Riscv64VectorCrypto` proves Zvkned availability for this key. + // SAFETY: `VectorCrypto` proves Zvkned availability for this key. unsafe { rv_aes::encrypt_4blocks_128(rk, batch) }; offset = offset.strict_add(4); } @@ -1853,17 +1823,17 @@ pub(crate) fn aes128_encrypt_blocks_ecb(ek: &Aes128EncKey, blocks: &mut [[u8; BL return; } #[cfg(target_arch = "riscv64")] - if let Key128Inner::Riscv64ScalarCrypto(rk) = &ek.inner { + if let Key128Inner::ScalarCrypto(rk) = &ek.inner { let mut offset = 0usize; while offset.strict_add(4) <= blocks.len() { let batch_slice = &mut blocks[offset..offset.strict_add(4)]; debug_assert_eq!(batch_slice.len(), 4); // SAFETY: exact four-block RISC-V Zkne AES-128 batch because: // 1. `batch_slice` is sliced to exactly four contiguous `[u8; 16]` elements. - // 2. `Riscv64ScalarCrypto` is only constructed after runtime detection confirms Zkne. + // 2. `ScalarCrypto` is only constructed after runtime detection confirms Zkne. // 3. The mutable borrow is scoped to this loop iteration. let batch: &mut [[u8; BLOCK_SIZE]; 4] = unsafe { &mut *batch_slice.as_mut_ptr().cast::<[[u8; BLOCK_SIZE]; 4]>() }; - // SAFETY: `Riscv64ScalarCrypto` proves Zkne availability for this key. + // SAFETY: `ScalarCrypto` proves Zkne availability for this key. unsafe { rv_scalar_aes::encrypt_4blocks_128(rk, batch) }; offset = offset.strict_add(4); } @@ -1875,26 +1845,7 @@ pub(crate) fn aes128_encrypt_blocks_ecb(ek: &Aes128EncKey, blocks: &mut [[u8; BL return; } #[cfg(target_arch = "riscv64")] - if let Key128Inner::Riscv64Vperm(rk) = &ek.inner { - let mut offset = 0usize; - while offset.strict_add(4) <= blocks.len() { - let batch_slice = &mut blocks[offset..offset.strict_add(4)]; - debug_assert_eq!(batch_slice.len(), 4); - // SAFETY: `batch_slice` is exactly 4 contiguous `[u8; 16]` elements. - let batch: &mut [[u8; BLOCK_SIZE]; 4] = unsafe { &mut *batch_slice.as_mut_ptr().cast::<[[u8; BLOCK_SIZE]; 4]>() }; - // SAFETY: Riscv64Vperm variant is only constructed after runtime detection confirms V extension. - unsafe { rv_vperm_aes::encrypt_4blocks_128(rk, batch) }; - offset = offset.strict_add(4); - } - while offset < blocks.len() { - // SAFETY: same V-extension guarantee as the wide path above. - unsafe { rv_vperm_aes::encrypt_block_128(rk, &mut blocks[offset]) }; - offset = offset.strict_add(1); - } - return; - } - #[cfg(target_arch = "riscv64")] - if let Key128Inner::Riscv64Fixslice(rk) = &ek.inner { + if let Key128Inner::Fixslice(rk) = &ek.inner { let mut offset = 0usize; while offset.strict_add(4) <= blocks.len() { let batch_slice = &mut blocks[offset..offset.strict_add(4)]; @@ -1928,6 +1879,7 @@ pub(crate) fn aes128_encrypt_blocks_ecb(ek: &Aes128EncKey, blocks: &mut [[u8; BL /// Portable AES-128 block encryption (10 rounds). #[inline] +#[cfg(any(not(target_arch = "riscv64"), test))] fn aes128_encrypt_block_portable(rk: &[u32; EXPANDED_KEY_WORDS_128], block: &mut [u8; BLOCK_SIZE]) { // Load state as four big-endian u32 columns. let mut s0 = u32::from_be_bytes([block[0], block[1], block[2], block[3]]); @@ -1973,17 +1925,6 @@ fn aes128_encrypt_block_portable(rk: &[u32; EXPANDED_KEY_WORDS_128], block: &mut /// On s390x this issues a single KM instruction for all `blocks`, /// avoiding per-block parameter-block setup overhead. On other platforms /// falls back to per-block dispatch. -#[cfg_attr( - not(any( - target_arch = "aarch64", - target_arch = "powerpc64", - target_arch = "riscv64", - target_arch = "s390x", - feature = "aes-gcm-siv", - test - )), - allow(dead_code) -)] #[inline] pub(crate) fn aes256_encrypt_blocks_ecb(ek: &Aes256EncKey, blocks: &mut [[u8; BLOCK_SIZE]]) { #[cfg(target_arch = "aarch64")] @@ -2021,17 +1962,17 @@ pub(crate) fn aes256_encrypt_blocks_ecb(ek: &Aes256EncKey, blocks: &mut [[u8; BL return; } #[cfg(target_arch = "riscv64")] - if let KeyInner::Riscv64VectorCrypto(rk) = &ek.inner { + if let KeyInner::VectorCrypto(rk) = &ek.inner { let mut offset = 0usize; while offset.strict_add(4) <= blocks.len() { let batch_slice = &mut blocks[offset..offset.strict_add(4)]; debug_assert_eq!(batch_slice.len(), 4); // SAFETY: exact four-block RISC-V Zvkned AES-256 batch because: // 1. `batch_slice` is sliced to exactly four contiguous `[u8; 16]` elements. - // 2. `Riscv64VectorCrypto` is only constructed after runtime detection confirms Zvkned. + // 2. `VectorCrypto` is only constructed after runtime detection confirms Zvkned. // 3. The mutable borrow is scoped to this loop iteration. let batch: &mut [[u8; BLOCK_SIZE]; 4] = unsafe { &mut *batch_slice.as_mut_ptr().cast::<[[u8; BLOCK_SIZE]; 4]>() }; - // SAFETY: `Riscv64VectorCrypto` proves Zvkned availability for this key. + // SAFETY: `VectorCrypto` proves Zvkned availability for this key. unsafe { rv_aes::encrypt_4blocks(rk, batch) }; offset = offset.strict_add(4); } @@ -2043,17 +1984,17 @@ pub(crate) fn aes256_encrypt_blocks_ecb(ek: &Aes256EncKey, blocks: &mut [[u8; BL return; } #[cfg(target_arch = "riscv64")] - if let KeyInner::Riscv64ScalarCrypto(rk) = &ek.inner { + if let KeyInner::ScalarCrypto(rk) = &ek.inner { let mut offset = 0usize; while offset.strict_add(4) <= blocks.len() { let batch_slice = &mut blocks[offset..offset.strict_add(4)]; debug_assert_eq!(batch_slice.len(), 4); // SAFETY: exact four-block RISC-V Zkne AES-256 batch because: // 1. `batch_slice` is sliced to exactly four contiguous `[u8; 16]` elements. - // 2. `Riscv64ScalarCrypto` is only constructed after runtime detection confirms Zkne. + // 2. `ScalarCrypto` is only constructed after runtime detection confirms Zkne. // 3. The mutable borrow is scoped to this loop iteration. let batch: &mut [[u8; BLOCK_SIZE]; 4] = unsafe { &mut *batch_slice.as_mut_ptr().cast::<[[u8; BLOCK_SIZE]; 4]>() }; - // SAFETY: `Riscv64ScalarCrypto` proves Zkne availability for this key. + // SAFETY: `ScalarCrypto` proves Zkne availability for this key. unsafe { rv_scalar_aes::encrypt_4blocks(rk, batch) }; offset = offset.strict_add(4); } @@ -2065,28 +2006,7 @@ pub(crate) fn aes256_encrypt_blocks_ecb(ek: &Aes256EncKey, blocks: &mut [[u8; BL return; } #[cfg(target_arch = "riscv64")] - if let KeyInner::Riscv64Vperm(rk) = &ek.inner { - let mut offset = 0usize; - while offset.strict_add(4) <= blocks.len() { - let batch_slice = &mut blocks[offset..offset.strict_add(4)]; - debug_assert_eq!(batch_slice.len(), 4); - // SAFETY: `batch_slice` is exactly 4 contiguous `[u8; 16]` elements, so - // reborrowing it as `&mut [[u8; 16]; 4]` preserves layout and bounds. - let batch: &mut [[u8; BLOCK_SIZE]; 4] = unsafe { &mut *batch_slice.as_mut_ptr().cast::<[[u8; BLOCK_SIZE]; 4]>() }; - // SAFETY: RvVperm variant is only constructed after runtime detection confirms V extension. - unsafe { rv_vperm_aes::encrypt_4blocks(rk, batch) }; - offset = offset.strict_add(4); - } - while offset < blocks.len() { - // SAFETY: Same runtime V-extension guarantee as above; tail stays on the - // existing single-block kernel to avoid special-casing 1-3 blocks. - unsafe { rv_vperm_aes::encrypt_block(rk, &mut blocks[offset]) }; - offset = offset.strict_add(1); - } - return; - } - #[cfg(target_arch = "riscv64")] - if let KeyInner::Riscv64Fixslice(rk) = &ek.inner { + if let KeyInner::Fixslice(rk) = &ek.inner { let mut offset = 0usize; while offset.strict_add(4) <= blocks.len() { let batch_slice = &mut blocks[offset..offset.strict_add(4)]; @@ -2120,6 +2040,7 @@ pub(crate) fn aes256_encrypt_blocks_ecb(ek: &Aes256EncKey, blocks: &mut [[u8; BL /// Portable AES-256 block encryption. #[inline] +#[cfg(any(not(target_arch = "riscv64"), test))] fn aes256_encrypt_block_portable(rk: &[u32; EXPANDED_KEY_WORDS], block: &mut [u8; BLOCK_SIZE]) { // Load state as four big-endian u32 columns. let mut s0 = u32::from_be_bytes([block[0], block[1], block[2], block[3]]); @@ -2162,12 +2083,14 @@ fn aes256_encrypt_block_portable(rk: &[u32; EXPANDED_KEY_WORDS], block: &mut [u8 /// Extract byte `row` from a big-endian column word. #[inline(always)] +#[cfg(any(not(target_arch = "riscv64"), test))] const fn col_byte(col: u32, row: usize) -> u8 { - (col >> (24u32.strict_sub((row as u32).strict_mul(8)))) as u8 + (col >> 24usize.strict_sub(row.strict_mul(8))).to_le_bytes()[0] } /// xtime: multiply by x in GF(2^8), i.e. x << 1 with conditional reduction. #[inline(always)] +#[cfg(any(not(target_arch = "riscv64"), test))] const fn xtime(x: u8) -> u8 { let hi = (x >> 7) & 1; (x << 1) ^ (hi.wrapping_mul(0x1b)) @@ -2178,6 +2101,7 @@ const fn xtime(x: u8) -> u8 { /// Input/output: four column words in big-endian byte order. /// AddRoundKey is done by the caller. #[inline(always)] +#[cfg(any(not(target_arch = "riscv64"), test))] const fn aes_round(s0: u32, s1: u32, s2: u32, s3: u32) -> (u32, u32, u32, u32) { // After SubBytes + ShiftRows, column j contains: // row 0 from column j, row 1 from (j+1)%4, row 2 from (j+2)%4, row 3 from (j+3)%4 @@ -2211,6 +2135,7 @@ const fn aes_round(s0: u32, s1: u32, s2: u32, s3: u32) -> (u32, u32, u32, u32) { /// Final AES round: SubBytes → ShiftRows (no MixColumns). #[inline(always)] +#[cfg(any(not(target_arch = "riscv64"), test))] const fn aes_final_round(s0: u32, s1: u32, s2: u32, s3: u32) -> (u32, u32, u32, u32) { let t0 = (sbox(col_byte(s0, 0)) as u32) << 24 | (sbox(col_byte(s1, 1)) as u32) << 16 @@ -2234,6 +2159,7 @@ const fn aes_final_round(s0: u32, s1: u32, s2: u32, s3: u32) -> (u32, u32, u32, /// MixColumns on a single column [b0, b1, b2, b3]. #[inline(always)] +#[cfg(any(not(target_arch = "riscv64"), test))] const fn mix_column(col: [u8; 4]) -> u32 { let [b0, b1, b2, b3] = col; @@ -2307,10 +2233,7 @@ pub(crate) fn aes256_ctr32_encrypt(ek: &Aes256EncKey, initial_counter: &[u8; BLO #[cfg(target_arch = "riscv64")] if matches!( &ek.inner, - KeyInner::Riscv64VectorCrypto(_) - | KeyInner::Riscv64ScalarCrypto(_) - | KeyInner::Riscv64Vperm(_) - | KeyInner::Riscv64Fixslice(_) + KeyInner::VectorCrypto(_) | KeyInner::ScalarCrypto(_) | KeyInner::Fixslice(_) ) { let iv_suffix: [u8; 12] = { let mut buf = [0u8; 12]; @@ -2367,7 +2290,7 @@ pub(crate) fn aes256_ctr32_encrypt(ek: &Aes256EncKey, initial_counter: &[u8; BLO aes256_encrypt_blocks_ecb(ek, &mut keystream[..block_count]); let processed = xor_keystream_tail(data, offset, &keystream, block_count); offset = offset.strict_add(processed); - ctr = ctr.wrapping_add(block_count as u32); + ctr = ctr.wrapping_add(u32::from(block_count.to_le_bytes()[0])); } } @@ -2416,10 +2339,7 @@ pub(crate) fn aes128_ctr32_encrypt(ek: &Aes128EncKey, initial_counter: &[u8; BLO #[cfg(target_arch = "riscv64")] if matches!( &ek.inner, - Key128Inner::Riscv64VectorCrypto(_) - | Key128Inner::Riscv64ScalarCrypto(_) - | Key128Inner::Riscv64Vperm(_) - | Key128Inner::Riscv64Fixslice(_) + Key128Inner::VectorCrypto(_) | Key128Inner::ScalarCrypto(_) | Key128Inner::Fixslice(_) ) { let iv_suffix: [u8; 12] = { let mut buf = [0u8; 12]; @@ -2476,7 +2396,7 @@ pub(crate) fn aes128_ctr32_encrypt(ek: &Aes128EncKey, initial_counter: &[u8; BLO aes128_encrypt_blocks_ecb(ek, &mut keystream[..block_count]); let processed = xor_keystream_tail(data, offset, &keystream, block_count); offset = offset.strict_add(processed); - ctr = ctr.wrapping_add(block_count as u32); + ctr = ctr.wrapping_add(u32::from(block_count.to_le_bytes()[0])); } } @@ -2528,10 +2448,8 @@ fn aes256_ctr32_be_uses_block_batch(ek: &Aes256EncKey) -> bool { #[cfg(target_arch = "s390x")] KeyInner::S390xMsa(_) => true, #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64VectorCrypto(_) - | KeyInner::Riscv64ScalarCrypto(_) - | KeyInner::Riscv64Vperm(_) - | KeyInner::Riscv64Fixslice(_) => true, + KeyInner::VectorCrypto(_) | KeyInner::ScalarCrypto(_) | KeyInner::Fixslice(_) => true, + #[cfg(not(target_arch = "riscv64"))] _ => false, } } @@ -2555,10 +2473,8 @@ fn aes128_ctr32_be_uses_block_batch(ek: &Aes128EncKey) -> bool { #[cfg(target_arch = "s390x")] Key128Inner::S390xMsa(_) => true, #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64VectorCrypto(_) - | Key128Inner::Riscv64ScalarCrypto(_) - | Key128Inner::Riscv64Vperm(_) - | Key128Inner::Riscv64Fixslice(_) => true, + Key128Inner::VectorCrypto(_) | Key128Inner::ScalarCrypto(_) | Key128Inner::Fixslice(_) => true, + #[cfg(not(target_arch = "riscv64"))] _ => false, } } @@ -2674,7 +2590,7 @@ pub(crate) fn aes256_ctr32_encrypt_be(ek: &Aes256EncKey, initial_counter: &[u8; aes256_encrypt_blocks_ecb(ek, &mut keystream[..block_count]); let processed = xor_keystream_tail(data, offset, &keystream, block_count); offset = offset.strict_add(processed); - ctr = ctr.wrapping_add(block_count as u32); + ctr = ctr.wrapping_add(u32::from(block_count.to_le_bytes()[0])); } } @@ -2724,13 +2640,13 @@ unsafe fn x86_gcm_ctr_blocks_be_4(iv_words: [u32; 3], ctr: u32) -> core::arch::x let c1 = ctr.wrapping_add(1).swap_bytes(); let c2 = ctr.wrapping_add(2).swap_bytes(); let c3 = ctr.wrapping_add(3).swap_bytes(); - let iv0 = iv_words[0] as i32; - let iv1 = iv_words[1] as i32; - let iv2 = iv_words[2] as i32; - let b0 = _mm_set_epi32(c0 as i32, iv2, iv1, iv0); - let b1 = _mm_set_epi32(c1 as i32, iv2, iv1, iv0); - let b2 = _mm_set_epi32(c2 as i32, iv2, iv1, iv0); - let b3 = _mm_set_epi32(c3 as i32, iv2, iv1, iv0); + let iv0 = iv_words[0].cast_signed(); + let iv1 = iv_words[1].cast_signed(); + let iv2 = iv_words[2].cast_signed(); + let b0 = _mm_set_epi32(c0.cast_signed(), iv2, iv1, iv0); + let b1 = _mm_set_epi32(c1.cast_signed(), iv2, iv1, iv0); + let b2 = _mm_set_epi32(c2.cast_signed(), iv2, iv1, iv0); + let b3 = _mm_set_epi32(c3.cast_signed(), iv2, iv1, iv0); let z = _mm512_zextsi128_si512(b0); let z = _mm512_inserti32x4(z, b1, 1); @@ -2738,6 +2654,10 @@ unsafe fn x86_gcm_ctr_blocks_be_4(iv_words: [u32; 3], ctr: u32) -> core::arch::x _mm512_inserti32x4(z, b3, 3) } +/// Build sixteen big-endian GCM counter blocks in four VAES registers. +/// +/// # Safety +/// Caller must ensure AVX-512F and AVX-512BW are available. #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] #[target_feature(enable = "avx512f,avx512bw")] #[inline] @@ -2752,9 +2672,14 @@ unsafe fn x86_gcm_ctr_blocks_be_16( ) { use core::arch::x86_64::*; - let iv = _mm_set_epi32(0, iv_words[2] as i32, iv_words[1] as i32, iv_words[0] as i32); + let iv = _mm_set_epi32( + 0, + iv_words[2].cast_signed(), + iv_words[1].cast_signed(), + iv_words[0].cast_signed(), + ); let template = _mm512_broadcast_i32x4(iv); - let ctrs = _mm512_set1_epi32(ctr as i32); + let ctrs = _mm512_set1_epi32(ctr.cast_signed()); let bswap = _mm512_broadcast_i32x4(_mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3)); let offsets0 = _mm512_set_epi32(3, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); @@ -2778,51 +2703,16 @@ unsafe fn x86_gcm_ctr_blocks_be_16( unsafe fn x86_gcm_ctr_blocks_be_2(iv_words: [u32; 3], ctr: u32) -> core::arch::x86_64::__m256i { use core::arch::x86_64::*; - let p0 = iv_words[0] as i32; - let p1 = iv_words[1] as i32; - let p2 = iv_words[2] as i32; - let b0 = _mm_set_epi32(ctr.to_be() as i32, p2, p1, p0); - let b1 = _mm_set_epi32(ctr.wrapping_add(1).to_be() as i32, p2, p1, p0); + let p0 = iv_words[0].cast_signed(); + let p1 = iv_words[1].cast_signed(); + let p2 = iv_words[2].cast_signed(); + let b0 = _mm_set_epi32(ctr.to_be().cast_signed(), p2, p1, p0); + let b1 = _mm_set_epi32(ctr.wrapping_add(1).to_be().cast_signed(), p2, p1, p0); let z = _mm256_castsi128_si256(b0); _mm256_inserti128_si256(z, b1, 1) } -#[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] -#[target_feature(enable = "avx2")] -#[inline] -#[allow(dead_code)] -unsafe fn x86_gcm_ctr_blocks_be_8_y256( - iv_words: [u32; 3], - ctr: u32, -) -> ( - core::arch::x86_64::__m256i, - core::arch::x86_64::__m256i, - core::arch::x86_64::__m256i, - core::arch::x86_64::__m256i, -) { - use core::arch::x86_64::*; - - let iv = _mm_set_epi32(0, iv_words[2] as i32, iv_words[1] as i32, iv_words[0] as i32); - let template = _mm256_broadcastsi128_si256(iv); - let ctrs = _mm256_set1_epi32(ctr as i32); - let bswap = _mm256_broadcastsi128_si256(_mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3)); - - let offsets0 = _mm256_set_epi32(1, 0, 0, 0, 0, 0, 0, 0); - let offsets1 = _mm256_set_epi32(3, 0, 0, 0, 2, 0, 0, 0); - let offsets2 = _mm256_set_epi32(5, 0, 0, 0, 4, 0, 0, 0); - let offsets3 = _mm256_set_epi32(7, 0, 0, 0, 6, 0, 0, 0); - - macro_rules! make { - ($offsets:expr) => {{ - let be = _mm256_shuffle_epi8(_mm256_add_epi32(ctrs, $offsets), bswap); - _mm256_blend_epi32::<0x88>(template, be) - }}; - } - - (make!(offsets0), make!(offsets1), make!(offsets2), make!(offsets3)) -} - /// Build one big-endian GCM counter block directly in an XMM register. /// /// # Safety @@ -2834,10 +2724,10 @@ unsafe fn x86_gcm_ctr_block_be(iv_words: [u32; 3], ctr: u32) -> core::arch::x86_ use core::arch::x86_64::*; _mm_set_epi32( - ctr.swap_bytes() as i32, - iv_words[2] as i32, - iv_words[1] as i32, - iv_words[0] as i32, + ctr.swap_bytes().cast_signed(), + iv_words[2].cast_signed(), + iv_words[1].cast_signed(), + iv_words[0].cast_signed(), ) } @@ -2851,13 +2741,13 @@ unsafe fn x86_gcm_ctr_block_be(iv_words: [u32; 3], ctr: u32) -> core::arch::x86_ unsafe fn x86_gcmsiv_ctr_blocks_le_4(suffix_words: [u32; 3], ctr: u32) -> core::arch::x86_64::__m512i { use core::arch::x86_64::*; - let s0 = suffix_words[0] as i32; - let s1 = suffix_words[1] as i32; - let s2 = suffix_words[2] as i32; - let b0 = _mm_set_epi32(s2, s1, s0, ctr as i32); - let b1 = _mm_set_epi32(s2, s1, s0, ctr.wrapping_add(1) as i32); - let b2 = _mm_set_epi32(s2, s1, s0, ctr.wrapping_add(2) as i32); - let b3 = _mm_set_epi32(s2, s1, s0, ctr.wrapping_add(3) as i32); + let s0 = suffix_words[0].cast_signed(); + let s1 = suffix_words[1].cast_signed(); + let s2 = suffix_words[2].cast_signed(); + let b0 = _mm_set_epi32(s2, s1, s0, ctr.cast_signed()); + let b1 = _mm_set_epi32(s2, s1, s0, ctr.wrapping_add(1).cast_signed()); + let b2 = _mm_set_epi32(s2, s1, s0, ctr.wrapping_add(2).cast_signed()); + let b3 = _mm_set_epi32(s2, s1, s0, ctr.wrapping_add(3).cast_signed()); let z = _mm512_zextsi128_si512(b0); let z = _mm512_inserti32x4(z, b1, 1); @@ -3184,7 +3074,7 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_128.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else if data.len() >= 1024 { x86_64_asm::rscrypto_aes256_gcm_seal_64x_vaes512_x86_64_linux( @@ -3193,7 +3083,7 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_64.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else { x86_64_asm::rscrypto_aes256_gcm_seal_16x_vaes512_x86_64_linux( @@ -3202,7 +3092,7 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_32.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } acc = state.acc(); @@ -3360,7 +3250,7 @@ pub(crate) unsafe fn aes256_ctr32_decrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_128.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else if data.len() >= 1024 { x86_64_asm::rscrypto_aes256_gcm_open_64x_vaes512_x86_64_linux( @@ -3369,7 +3259,7 @@ pub(crate) unsafe fn aes256_ctr32_decrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_64.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else { x86_64_asm::rscrypto_aes256_gcm_open_16x_vaes512_x86_64_linux( @@ -3378,7 +3268,7 @@ pub(crate) unsafe fn aes256_ctr32_decrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_32.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } acc = state.acc(); @@ -3465,39 +3355,33 @@ pub(crate) unsafe fn aes256_ctr32_decrypt_be_wide_ghash( } } -/// AES-256 CTR encryption fused with 256-bit VAES/VPCLMUL GHASH accumulation. +/// AES-256 CTR encryption fused with PMULL GHASH accumulation for GCM sealing. /// -/// This avoids ZMM data-path pressure on AMD while preserving the fused -/// counter/AES/XOR/GHASH structure used by the 512-bit path. +/// Encrypts `data` in place and returns the GHASH accumulator after the +/// ciphertext has been folded in. The incoming `acc` is normally the GHASH +/// state after AAD processing. /// /// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VAES + VPCLMULQDQ + -/// PCLMULQDQ + AES + SSE2 + SSSE3. -#[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] -#[target_feature(enable = "aes,sse2,ssse3,avx2,avx512f,avx512vl,vaes,vpclmulqdq,pclmulqdq")] -#[allow(dead_code)] -pub(crate) unsafe fn aes256_ctr32_encrypt_be_y256_ghash( +/// Caller must ensure AES-CE and PMULL are available. +#[cfg(all(target_arch = "aarch64", feature = "aes-gcm"))] +#[target_feature(enable = "aes,neon")] +pub(crate) unsafe fn aes256_ctr32_encrypt_be_aarch64_ghash( ek: &Aes256EncKey, initial_counter: &[u8; BLOCK_SIZE], data: &mut [u8], mut acc: u128, - h_polyval: u128, - h_powers_rev: &[u128; 4], - h_powers_rev_8: &[u128; 8], + tables: &Aarch64GcmTables<'_>, ) -> u128 { - use core::arch::x86_64::*; - - // SAFETY: fused x86 VAES-256 AES-GCM sealing because: - // 1. This function's caller guarantees all required x86 target features. - // 2. `data` is a valid mutable byte slice; all pointer arithmetic stays inside checked chunk - // bounds. - // 3. GHASH folds ciphertext registers after encryption, matching GCM authentication semantics. + // SAFETY: fused aarch64 AES-GCM sealing because: + // 1. This function's caller guarantees AES-CE and PMULL availability. + // 2. `data` is a valid mutable byte slice; all chunk processing uses checked slice ranges and the + // tail path handles partial blocks. unsafe { - let ni_rk = match &ek.inner { - KeyInner::X86AesNi(rk) => rk, + let ce_rk = match &ek.inner { + KeyInner::Aarch64Aes(rk) => rk, _ => { aes256_ctr32_encrypt_be(ek, initial_counter, data); - return ghash_ciphertext_fallback(acc, h_polyval, data); + return ghash_ciphertext_fallback(acc, tables.h_polyval, data); } }; @@ -3506,11 +3390,6 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_y256_ghash( buf.copy_from_slice(&initial_counter[..12]); buf }; - let iv_words = [ - u32::from_le_bytes([iv_prefix[0], iv_prefix[1], iv_prefix[2], iv_prefix[3]]), - u32::from_le_bytes([iv_prefix[4], iv_prefix[5], iv_prefix[6], iv_prefix[7]]), - u32::from_le_bytes([iv_prefix[8], iv_prefix[9], iv_prefix[10], iv_prefix[11]]), - ]; let mut ctr = u32::from_be_bytes([ initial_counter[12], initial_counter[13], @@ -3519,89 +3398,28 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_y256_ghash( ]); let mut offset = 0usize; - #[cfg(target_os = "linux")] - if data.len() >= 128 { - let mut state = x86_64_asm::AesGcmX86State::new(acc, ctr); - // SAFETY: external x86-64 VAES-256 AES-256-GCM seal kernel because: - // 1. This target-feature function is only entered after VAES, VPCLMULQDQ, AVX2, AVX-512VL, AES-NI, - // PCLMULQDQ, SSE2, and SSSE3 were selected by runtime/backend dispatch. - // 2. `ni_rk.as_ptr()` addresses 15 initialized 128-bit AES-256 round keys. - // 3. `initial_counter` points to the full 16-byte GCM counter block, and `data` is valid for - // `data.len()` mutable bytes. - // 4. `h_powers_rev_8` contains exactly the [H^8..H] powers required by the 8-block fold. - // 5. The kernel only processes complete 128-byte chunks and reports the processed byte count so the - // Rust fallback below handles every remaining full/partial tail. - x86_64_asm::rscrypto_aes256_gcm_seal_8x_vaes256_x86_64_linux( - ni_rk.as_ptr(), - initial_counter.as_ptr(), - data.as_mut_ptr(), - data.len(), - h_powers_rev_8.as_ptr(), - &mut state, - ); - acc = state.acc(); - ctr = state.ctr; - offset = state.processed; - } - - while offset.strict_add(128) <= data.len() { - let (ctr0, ctr1, ctr2, ctr3) = x86_gcm_ctr_blocks_be_8_y256(iv_words, ctr); - let (ks0, ks1, ks2, ks3) = ni::encrypt_8blocks_y256(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let p0 = _mm256_loadu_si256(data.as_ptr().add(offset).cast()); - let c0 = _mm256_xor_si256(p0, ks0); - _mm256_storeu_si256(data.as_mut_ptr().add(offset).cast(), c0); - - let p1 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(32)).cast()); - let c1 = _mm256_xor_si256(p1, ks1); - _mm256_storeu_si256(data.as_mut_ptr().add(offset.strict_add(32)).cast(), c1); - - let p2 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(64)).cast()); - let c2 = _mm256_xor_si256(p2, ks2); - _mm256_storeu_si256(data.as_mut_ptr().add(offset.strict_add(64)).cast(), c2); - - let p3 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(96)).cast()); - let c3 = _mm256_xor_si256(p3, ks3); - _mm256_storeu_si256(data.as_mut_ptr().add(offset.strict_add(96)).cast(), c3); - acc = super::polyval::x86_aggregate_8blocks_be_lanes_256_inline(acc, h_powers_rev_8, c0, c1, c2, c3); - - ctr = ctr.wrapping_add(8); - offset = offset.strict_add(128); + if offset.strict_add(128) <= data.len() { + let state = ce::encrypt_ctr32_be_xor_ghash_128b_chunks_core(ce_rk, &iv_prefix, ctr, data, acc, tables); + acc = state.0; + ctr = state.1; + offset = state.2; } while offset.strict_add(64) <= data.len() { - let ctr0 = x86_gcm_ctr_block_be(iv_words, ctr); - let ctr1 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(1)); - let ctr2 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(2)); - let ctr3 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(3)); - let (ks0, ks1, ks2, ks3) = ni::encrypt_4blocks_aesni(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let ptr = data.as_mut_ptr().add(offset); - let p0 = _mm_loadu_si128(ptr.cast()); - let p1 = _mm_loadu_si128(ptr.add(16).cast()); - let p2 = _mm_loadu_si128(ptr.add(32).cast()); - let p3 = _mm_loadu_si128(ptr.add(48).cast()); - let c0 = _mm_xor_si128(p0, ks0); - let c1 = _mm_xor_si128(p1, ks1); - let c2 = _mm_xor_si128(p2, ks2); - let c3 = _mm_xor_si128(p3, ks3); - _mm_storeu_si128(ptr.cast(), c0); - _mm_storeu_si128(ptr.add(16).cast(), c1); - _mm_storeu_si128(ptr.add(32).cast(), c2); - _mm_storeu_si128(ptr.add(48).cast(), c3); - acc = super::polyval::x86_pclmul_aggregate_4blocks_be_xmm_inline(acc, h_powers_rev, c0, c1, c2, c3); - + let end = offset.strict_add(64); + let blocks = ce::encrypt_ctr32_be_xor_4blocks_core(ce_rk, &iv_prefix, ctr, &mut data[offset..end]); + acc = super::polyval::aarch64_aggregate_4blocks_inline(acc, tables.h_powers_rev, &blocks); ctr = ctr.wrapping_add(4); - offset = offset.strict_add(64); + offset = end; } while offset < data.len() { - let mut counter_block = [0u8; 16]; + let mut counter_block = [0u8; BLOCK_SIZE]; counter_block[..12].copy_from_slice(&iv_prefix); counter_block[12..16].copy_from_slice(&ctr.to_be_bytes()); let mut keystream = counter_block; - ni::encrypt_block(ni_rk, &mut keystream); + ce::encrypt_block_core(ce_rk, &mut keystream); let remaining = data.len().strict_sub(offset); if remaining >= BLOCK_SIZE { @@ -3612,7 +3430,7 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_y256_ghash( let ciphertext = xored.to_ne_bytes(); data[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&ciphertext); acc ^= u128::from_be_bytes(ciphertext); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); + acc = super::polyval::aarch64_clmul128_reduce_inline(acc, tables.h_polyval); offset = offset.strict_add(BLOCK_SIZE); } else { let mut block = [0u8; BLOCK_SIZE]; @@ -3623,7 +3441,7 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_y256_ghash( i = i.strict_add(1); } acc ^= u128::from_be_bytes(block); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); + acc = super::polyval::aarch64_clmul128_reduce_inline(acc, tables.h_polyval); offset = offset.strict_add(remaining); } ctr = ctr.wrapping_add(1); @@ -3633,34 +3451,33 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_y256_ghash( } } -/// AES-256 CTR decryption fused with 256-bit VAES/VPCLMUL GHASH accumulation. +/// AES-256 CTR decryption fused with PMULL GHASH accumulation for GCM open. +/// +/// Folds ciphertext into GHASH, decrypts the same chunk in place, and returns +/// the accumulator. Authentication is still finalized and checked by the +/// caller, which zeroizes the buffer on failure. /// /// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VAES + VPCLMULQDQ + -/// PCLMULQDQ + AES + SSE2 + SSSE3. -#[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] -#[target_feature(enable = "aes,sse2,ssse3,avx2,avx512f,avx512vl,vaes,vpclmulqdq,pclmulqdq")] -#[allow(dead_code)] -pub(crate) unsafe fn aes256_ctr32_decrypt_be_y256_ghash( +/// Caller must ensure AES-CE and PMULL are available. +#[cfg(all(target_arch = "aarch64", feature = "aes-gcm"))] +#[target_feature(enable = "aes,neon")] +pub(crate) unsafe fn aes256_ctr32_decrypt_be_aarch64_ghash( ek: &Aes256EncKey, initial_counter: &[u8; BLOCK_SIZE], data: &mut [u8], mut acc: u128, - h_polyval: u128, - h_powers_rev: &[u128; 4], - h_powers_rev_8: &[u128; 8], + tables: &Aarch64GcmTables<'_>, ) -> u128 { - use core::arch::x86_64::*; - - // SAFETY: fused x86 VAES-256 AES-GCM opening because: - // 1. This function's caller guarantees all required x86 target features. - // 2. Ciphertext registers are folded into GHASH before plaintext is stored back. - // 3. All pointer arithmetic stays inside checked chunk bounds. + // SAFETY: fused aarch64 AES-GCM opening because: + // 1. This function's caller guarantees AES-CE and PMULL availability. + // 2. `data` is a valid mutable ciphertext slice. Ciphertext blocks are copied before in-place + // decryption, so GHASH always authenticates the original bytes. + // 3. All chunk processing uses checked slice bounds and the tail path handles partial blocks. unsafe { - let ni_rk = match &ek.inner { - KeyInner::X86AesNi(rk) => rk, + let ce_rk = match &ek.inner { + KeyInner::Aarch64Aes(rk) => rk, _ => { - acc = ghash_ciphertext_fallback(acc, h_polyval, data); + acc = ghash_ciphertext_fallback(acc, tables.h_polyval, data); aes256_ctr32_encrypt_be(ek, initial_counter, data); return acc; } @@ -3671,275 +3488,16 @@ pub(crate) unsafe fn aes256_ctr32_decrypt_be_y256_ghash( buf.copy_from_slice(&initial_counter[..12]); buf }; - let iv_words = [ - u32::from_le_bytes([iv_prefix[0], iv_prefix[1], iv_prefix[2], iv_prefix[3]]), - u32::from_le_bytes([iv_prefix[4], iv_prefix[5], iv_prefix[6], iv_prefix[7]]), - u32::from_le_bytes([iv_prefix[8], iv_prefix[9], iv_prefix[10], iv_prefix[11]]), - ]; let mut ctr = u32::from_be_bytes([ initial_counter[12], initial_counter[13], initial_counter[14], initial_counter[15], ]); - let mut offset = 0usize; - - #[cfg(target_os = "linux")] - if data.len() >= 128 { - let mut state = x86_64_asm::AesGcmX86State::new(acc, ctr); - // SAFETY: external x86-64 VAES-256 AES-256-GCM open kernel because: - // 1. This target-feature function is only entered after VAES, VPCLMULQDQ, AVX2, AVX-512VL, AES-NI, - // PCLMULQDQ, SSE2, and SSSE3 were selected by runtime/backend dispatch. - // 2. `ni_rk.as_ptr()` addresses 15 initialized 128-bit AES-256 round keys. - // 3. `initial_counter` points to the full 16-byte GCM counter block, and `data` is valid for - // `data.len()` mutable ciphertext bytes. - // 4. The kernel folds ciphertext into GHASH before storing plaintext. - // 5. The kernel only processes complete 128-byte chunks and reports the processed byte count so the - // Rust fallback below handles every remaining full/partial tail. - x86_64_asm::rscrypto_aes256_gcm_open_8x_vaes256_x86_64_linux( - ni_rk.as_ptr(), - initial_counter.as_ptr(), - data.as_mut_ptr(), - data.len(), - h_powers_rev_8.as_ptr(), - &mut state, - ); - acc = state.acc(); - ctr = state.ctr; - offset = state.processed; - } - - while offset.strict_add(128) <= data.len() { - let (ctr0, ctr1, ctr2, ctr3) = x86_gcm_ctr_blocks_be_8_y256(iv_words, ctr); - let (ks0, ks1, ks2, ks3) = ni::encrypt_8blocks_y256(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let c0 = _mm256_loadu_si256(data.as_ptr().add(offset).cast()); - let c1 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(32)).cast()); - let c2 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(64)).cast()); - let c3 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(96)).cast()); - acc = super::polyval::x86_aggregate_8blocks_be_lanes_256_inline(acc, h_powers_rev_8, c0, c1, c2, c3); - - _mm256_storeu_si256(data.as_mut_ptr().add(offset).cast(), _mm256_xor_si256(c0, ks0)); - _mm256_storeu_si256( - data.as_mut_ptr().add(offset.strict_add(32)).cast(), - _mm256_xor_si256(c1, ks1), - ); - _mm256_storeu_si256( - data.as_mut_ptr().add(offset.strict_add(64)).cast(), - _mm256_xor_si256(c2, ks2), - ); - _mm256_storeu_si256( - data.as_mut_ptr().add(offset.strict_add(96)).cast(), - _mm256_xor_si256(c3, ks3), - ); - - ctr = ctr.wrapping_add(8); - offset = offset.strict_add(128); - } - - while offset.strict_add(64) <= data.len() { - let ctr0 = x86_gcm_ctr_block_be(iv_words, ctr); - let ctr1 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(1)); - let ctr2 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(2)); - let ctr3 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(3)); - let (ks0, ks1, ks2, ks3) = ni::encrypt_4blocks_aesni(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let ptr = data.as_mut_ptr().add(offset); - let c0 = _mm_loadu_si128(ptr.cast()); - let c1 = _mm_loadu_si128(ptr.add(16).cast()); - let c2 = _mm_loadu_si128(ptr.add(32).cast()); - let c3 = _mm_loadu_si128(ptr.add(48).cast()); - acc = super::polyval::x86_pclmul_aggregate_4blocks_be_xmm_inline(acc, h_powers_rev, c0, c1, c2, c3); - _mm_storeu_si128(ptr.cast(), _mm_xor_si128(c0, ks0)); - _mm_storeu_si128(ptr.add(16).cast(), _mm_xor_si128(c1, ks1)); - _mm_storeu_si128(ptr.add(32).cast(), _mm_xor_si128(c2, ks2)); - _mm_storeu_si128(ptr.add(48).cast(), _mm_xor_si128(c3, ks3)); - - ctr = ctr.wrapping_add(4); - offset = offset.strict_add(64); - } - - while offset < data.len() { - let mut counter_block = [0u8; 16]; - counter_block[..12].copy_from_slice(&iv_prefix); - counter_block[12..16].copy_from_slice(&ctr.to_be_bytes()); - - let mut keystream = counter_block; - ni::encrypt_block(ni_rk, &mut keystream); - - let remaining = data.len().strict_sub(offset); - if remaining >= BLOCK_SIZE { - let mut ciphertext = [0u8; BLOCK_SIZE]; - ciphertext.copy_from_slice(&data[offset..offset.strict_add(BLOCK_SIZE)]); - acc ^= u128::from_be_bytes(ciphertext); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); - - let plaintext = u128::from_ne_bytes(ciphertext) ^ u128::from_ne_bytes(keystream); - data[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&plaintext.to_ne_bytes()); - offset = offset.strict_add(BLOCK_SIZE); - } else { - let mut block = [0u8; BLOCK_SIZE]; - block[..remaining].copy_from_slice(&data[offset..offset.strict_add(remaining)]); - acc ^= u128::from_be_bytes(block); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); - - let mut i = 0usize; - while i < remaining { - data[offset.strict_add(i)] ^= keystream[i]; - i = i.strict_add(1); - } - offset = offset.strict_add(remaining); - } - ctr = ctr.wrapping_add(1); - } - - acc - } -} - -/// AES-256 CTR encryption fused with PMULL GHASH accumulation for GCM sealing. -/// -/// Encrypts `data` in place and returns the GHASH accumulator after the -/// ciphertext has been folded in. The incoming `acc` is normally the GHASH -/// state after AAD processing. -/// -/// # Safety -/// Caller must ensure AES-CE and PMULL are available. -#[cfg(all(target_arch = "aarch64", feature = "aes-gcm"))] -#[target_feature(enable = "aes,neon")] -pub(crate) unsafe fn aes256_ctr32_encrypt_be_aarch64_ghash( - ek: &Aes256EncKey, - initial_counter: &[u8; BLOCK_SIZE], - data: &mut [u8], - mut acc: u128, - tables: &Aarch64GcmTables<'_>, -) -> u128 { - // SAFETY: fused aarch64 AES-GCM sealing because: - // 1. This function's caller guarantees AES-CE and PMULL availability. - // 2. `data` is a valid mutable byte slice; all chunk processing uses checked slice ranges and the - // tail path handles partial blocks. - unsafe { - let ce_rk = match &ek.inner { - KeyInner::Aarch64Aes(rk) => rk, - _ => { - aes256_ctr32_encrypt_be(ek, initial_counter, data); - return ghash_ciphertext_fallback(acc, tables.h_polyval, data); - } - }; - - let iv_prefix: [u8; 12] = { - let mut buf = [0u8; 12]; - buf.copy_from_slice(&initial_counter[..12]); - buf - }; - let mut ctr = u32::from_be_bytes([ - initial_counter[12], - initial_counter[13], - initial_counter[14], - initial_counter[15], - ]); - let mut offset = 0usize; - - if offset.strict_add(128) <= data.len() { - let state = ce::encrypt_ctr32_be_xor_ghash_128b_chunks_core(ce_rk, &iv_prefix, ctr, data, acc, tables); - acc = state.0; - ctr = state.1; - offset = state.2; - } - - while offset.strict_add(64) <= data.len() { - let end = offset.strict_add(64); - let blocks = ce::encrypt_ctr32_be_xor_4blocks_core(ce_rk, &iv_prefix, ctr, &mut data[offset..end]); - acc = super::polyval::aarch64_aggregate_4blocks_inline(acc, tables.h_powers_rev, &blocks); - ctr = ctr.wrapping_add(4); - offset = end; - } - - while offset < data.len() { - let mut counter_block = [0u8; BLOCK_SIZE]; - counter_block[..12].copy_from_slice(&iv_prefix); - counter_block[12..16].copy_from_slice(&ctr.to_be_bytes()); - - let mut keystream = counter_block; - ce::encrypt_block_core(ce_rk, &mut keystream); - - let remaining = data.len().strict_sub(offset); - if remaining >= BLOCK_SIZE { - let ks = u128::from_ne_bytes(keystream); - let mut d = [0u8; BLOCK_SIZE]; - d.copy_from_slice(&data[offset..offset.strict_add(BLOCK_SIZE)]); - let xored = u128::from_ne_bytes(d) ^ ks; - let ciphertext = xored.to_ne_bytes(); - data[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&ciphertext); - acc ^= u128::from_be_bytes(ciphertext); - acc = super::polyval::aarch64_clmul128_reduce_inline(acc, tables.h_polyval); - offset = offset.strict_add(BLOCK_SIZE); - } else { - let mut block = [0u8; BLOCK_SIZE]; - let mut i = 0usize; - while i < remaining { - data[offset.strict_add(i)] ^= keystream[i]; - block[i] = data[offset.strict_add(i)]; - i = i.strict_add(1); - } - acc ^= u128::from_be_bytes(block); - acc = super::polyval::aarch64_clmul128_reduce_inline(acc, tables.h_polyval); - offset = offset.strict_add(remaining); - } - ctr = ctr.wrapping_add(1); - } - - acc - } -} - -/// AES-256 CTR decryption fused with PMULL GHASH accumulation for GCM open. -/// -/// Folds ciphertext into GHASH, decrypts the same chunk in place, and returns -/// the accumulator. Authentication is still finalized and checked by the -/// caller, which zeroizes the buffer on failure. -/// -/// # Safety -/// Caller must ensure AES-CE and PMULL are available. -#[cfg(all(target_arch = "aarch64", feature = "aes-gcm"))] -#[target_feature(enable = "aes,neon")] -pub(crate) unsafe fn aes256_ctr32_decrypt_be_aarch64_ghash( - ek: &Aes256EncKey, - initial_counter: &[u8; BLOCK_SIZE], - data: &mut [u8], - mut acc: u128, - tables: &Aarch64GcmTables<'_>, -) -> u128 { - // SAFETY: fused aarch64 AES-GCM opening because: - // 1. This function's caller guarantees AES-CE and PMULL availability. - // 2. `data` is a valid mutable ciphertext slice. Ciphertext blocks are copied before in-place - // decryption, so GHASH always authenticates the original bytes. - // 3. All chunk processing uses checked slice bounds and the tail path handles partial blocks. - unsafe { - let ce_rk = match &ek.inner { - KeyInner::Aarch64Aes(rk) => rk, - _ => { - acc = ghash_ciphertext_fallback(acc, tables.h_polyval, data); - aes256_ctr32_encrypt_be(ek, initial_counter, data); - return acc; - } - }; - - let iv_prefix: [u8; 12] = { - let mut buf = [0u8; 12]; - buf.copy_from_slice(&initial_counter[..12]); - buf - }; - let mut ctr = u32::from_be_bytes([ - initial_counter[12], - initial_counter[13], - initial_counter[14], - initial_counter[15], - ]); - let state = ce::decrypt_ctr32_be_xor_ghash_128b_chunks_core(ce_rk, &iv_prefix, ctr, data, acc, tables); - acc = state.0; - ctr = state.1; - let mut offset = state.2; + let state = ce::decrypt_ctr32_be_xor_ghash_128b_chunks_core(ce_rk, &iv_prefix, ctr, data, acc, tables); + acc = state.0; + ctr = state.1; + let mut offset = state.2; while offset.strict_add(64) <= data.len() { let end = offset.strict_add(64); @@ -4108,7 +3666,7 @@ pub(crate) fn aes128_ctr32_encrypt_be(ek: &Aes128EncKey, initial_counter: &[u8; aes128_encrypt_blocks_ecb(ek, &mut keystream[..block_count]); let processed = xor_keystream_tail(data, offset, &keystream, block_count); offset = offset.strict_add(processed); - ctr = ctr.wrapping_add(block_count as u32); + ctr = ctr.wrapping_add(u32::from(block_count.to_le_bytes()[0])); } } @@ -4641,7 +4199,7 @@ pub(crate) unsafe fn aes128_ctr32_encrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_128.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else if data.len() >= 1024 { x86_64_asm::rscrypto_aes128_gcm_seal_64x_vaes512_x86_64_linux( @@ -4650,7 +4208,7 @@ pub(crate) unsafe fn aes128_ctr32_encrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_64.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else { x86_64_asm::rscrypto_aes128_gcm_seal_16x_vaes512_x86_64_linux( @@ -4659,7 +4217,7 @@ pub(crate) unsafe fn aes128_ctr32_encrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_32.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } acc = state.acc(); @@ -4817,7 +4375,7 @@ pub(crate) unsafe fn aes128_ctr32_decrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_128.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else if data.len() >= 1024 { x86_64_asm::rscrypto_aes128_gcm_open_64x_vaes512_x86_64_linux( @@ -4826,7 +4384,7 @@ pub(crate) unsafe fn aes128_ctr32_decrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_64.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else { x86_64_asm::rscrypto_aes128_gcm_open_16x_vaes512_x86_64_linux( @@ -4835,379 +4393,50 @@ pub(crate) unsafe fn aes128_ctr32_decrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_32.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); - } - acc = state.acc(); - ctr = state.ctr; - offset = state.processed; - } - - while offset.strict_add(256) <= data.len() { - let (ctr0, ctr1, ctr2, ctr3) = x86_gcm_ctr_blocks_be_16(iv_words, ctr); - let (ks0, ks1, ks2, ks3) = ni::encrypt_16blocks_128(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let c0 = _mm512_loadu_si512(data.as_ptr().add(offset).cast()); - let c1 = _mm512_loadu_si512(data.as_ptr().add(offset.strict_add(64)).cast()); - let c2 = _mm512_loadu_si512(data.as_ptr().add(offset.strict_add(128)).cast()); - let c3 = _mm512_loadu_si512(data.as_ptr().add(offset.strict_add(192)).cast()); - acc = super::polyval::x86_aggregate_16blocks_be_lanes_inline(acc, h_powers_rev_16, c0, c1, c2, c3); - _mm512_storeu_si512(data.as_mut_ptr().add(offset).cast(), _mm512_xor_si512(c0, ks0)); - _mm512_storeu_si512( - data.as_mut_ptr().add(offset.strict_add(64)).cast(), - _mm512_xor_si512(c1, ks1), - ); - _mm512_storeu_si512( - data.as_mut_ptr().add(offset.strict_add(128)).cast(), - _mm512_xor_si512(c2, ks2), - ); - _mm512_storeu_si512( - data.as_mut_ptr().add(offset.strict_add(192)).cast(), - _mm512_xor_si512(c3, ks3), - ); - - ctr = ctr.wrapping_add(16); - offset = offset.strict_add(256); - } - - while offset.strict_add(64) <= data.len() { - let ctr_vec = x86_gcm_ctr_blocks_be_4(iv_words, ctr); - let keystream = ni::encrypt_4blocks_128(ni_rk, ctr_vec); - let ciphertext = _mm512_loadu_si512(data.as_ptr().add(offset).cast()); - acc = super::polyval::x86_aggregate_4blocks_be_lanes_inline(acc, h_powers_rev, ciphertext); - _mm512_storeu_si512( - data.as_mut_ptr().add(offset).cast(), - _mm512_xor_si512(ciphertext, keystream), - ); - - ctr = ctr.wrapping_add(4); - offset = offset.strict_add(64); - } - - while offset < data.len() { - let mut counter_block = [0u8; 16]; - counter_block[..12].copy_from_slice(&iv_prefix); - counter_block[12..16].copy_from_slice(&ctr.to_be_bytes()); - - let mut keystream = counter_block; - ni::encrypt_block_128(ni_rk, &mut keystream); - - let remaining = data.len().strict_sub(offset); - if remaining >= BLOCK_SIZE { - let mut ciphertext = [0u8; BLOCK_SIZE]; - ciphertext.copy_from_slice(&data[offset..offset.strict_add(BLOCK_SIZE)]); - acc ^= u128::from_be_bytes(ciphertext); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); - - let plaintext = u128::from_ne_bytes(ciphertext) ^ u128::from_ne_bytes(keystream); - data[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&plaintext.to_ne_bytes()); - offset = offset.strict_add(BLOCK_SIZE); - } else { - let mut block = [0u8; BLOCK_SIZE]; - block[..remaining].copy_from_slice(&data[offset..offset.strict_add(remaining)]); - acc ^= u128::from_be_bytes(block); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); - - let mut i = 0usize; - while i < remaining { - data[offset.strict_add(i)] ^= keystream[i]; - i = i.strict_add(1); - } - offset = offset.strict_add(remaining); - } - ctr = ctr.wrapping_add(1); - } - - acc - } -} - -/// AES-128 CTR encryption fused with 256-bit VAES/VPCLMUL GHASH accumulation. -/// -/// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VAES + VPCLMULQDQ + -/// PCLMULQDQ + AES + SSE2 + SSSE3. -#[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] -#[target_feature(enable = "aes,sse2,ssse3,avx2,avx512f,avx512vl,vaes,vpclmulqdq,pclmulqdq")] -#[allow(dead_code)] -pub(crate) unsafe fn aes128_ctr32_encrypt_be_y256_ghash( - ek: &Aes128EncKey, - initial_counter: &[u8; BLOCK_SIZE], - data: &mut [u8], - mut acc: u128, - h_polyval: u128, - h_powers_rev: &[u128; 4], - h_powers_rev_8: &[u128; 8], -) -> u128 { - use core::arch::x86_64::*; - - // SAFETY: fused x86 VAES-128 AES-GCM sealing because: - // 1. This function's caller guarantees all required x86 target features. - // 2. `data` is a valid mutable byte slice; all pointer arithmetic stays inside checked chunk - // bounds. - // 3. GHASH folds ciphertext registers after encryption, matching GCM authentication semantics. - unsafe { - let ni_rk = match &ek.inner { - Key128Inner::X86AesNi(rk) => rk, - _ => { - aes128_ctr32_encrypt_be(ek, initial_counter, data); - return ghash_ciphertext_fallback(acc, h_polyval, data); - } - }; - - let iv_prefix: [u8; 12] = { - let mut buf = [0u8; 12]; - buf.copy_from_slice(&initial_counter[..12]); - buf - }; - let iv_words = [ - u32::from_le_bytes([iv_prefix[0], iv_prefix[1], iv_prefix[2], iv_prefix[3]]), - u32::from_le_bytes([iv_prefix[4], iv_prefix[5], iv_prefix[6], iv_prefix[7]]), - u32::from_le_bytes([iv_prefix[8], iv_prefix[9], iv_prefix[10], iv_prefix[11]]), - ]; - let mut ctr = u32::from_be_bytes([ - initial_counter[12], - initial_counter[13], - initial_counter[14], - initial_counter[15], - ]); - let mut offset = 0usize; - - #[cfg(target_os = "linux")] - if data.len() >= 128 { - let mut state = x86_64_asm::AesGcmX86State::new(acc, ctr); - // SAFETY: external x86-64 VAES-256 AES-128-GCM seal kernel because: - // 1. This target-feature function is only entered after VAES, VPCLMULQDQ, AVX2, AVX-512VL, AES-NI, - // PCLMULQDQ, SSE2, and SSSE3 were selected by runtime/backend dispatch. - // 2. `ni_rk.as_ptr()` addresses 11 initialized 128-bit AES-128 round keys. - // 3. `initial_counter` points to the full 16-byte GCM counter block, and `data` is valid for - // `data.len()` mutable bytes. - // 4. `h_powers_rev_8` contains exactly the [H^8..H] powers required by the 8-block fold. - // 5. The kernel only processes complete 128-byte chunks and reports the processed byte count so the - // Rust fallback below handles every remaining full/partial tail. - x86_64_asm::rscrypto_aes128_gcm_seal_8x_vaes256_x86_64_linux( - ni_rk.as_ptr(), - initial_counter.as_ptr(), - data.as_mut_ptr(), - data.len(), - h_powers_rev_8.as_ptr(), - &mut state, - ); - acc = state.acc(); - ctr = state.ctr; - offset = state.processed; - } - - while offset.strict_add(128) <= data.len() { - let (ctr0, ctr1, ctr2, ctr3) = x86_gcm_ctr_blocks_be_8_y256(iv_words, ctr); - let (ks0, ks1, ks2, ks3) = ni::encrypt_8blocks_128_y256(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let p0 = _mm256_loadu_si256(data.as_ptr().add(offset).cast()); - let c0 = _mm256_xor_si256(p0, ks0); - _mm256_storeu_si256(data.as_mut_ptr().add(offset).cast(), c0); - - let p1 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(32)).cast()); - let c1 = _mm256_xor_si256(p1, ks1); - _mm256_storeu_si256(data.as_mut_ptr().add(offset.strict_add(32)).cast(), c1); - - let p2 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(64)).cast()); - let c2 = _mm256_xor_si256(p2, ks2); - _mm256_storeu_si256(data.as_mut_ptr().add(offset.strict_add(64)).cast(), c2); - - let p3 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(96)).cast()); - let c3 = _mm256_xor_si256(p3, ks3); - _mm256_storeu_si256(data.as_mut_ptr().add(offset.strict_add(96)).cast(), c3); - acc = super::polyval::x86_aggregate_8blocks_be_lanes_256_inline(acc, h_powers_rev_8, c0, c1, c2, c3); - - ctr = ctr.wrapping_add(8); - offset = offset.strict_add(128); - } - - while offset.strict_add(64) <= data.len() { - let ctr0 = x86_gcm_ctr_block_be(iv_words, ctr); - let ctr1 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(1)); - let ctr2 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(2)); - let ctr3 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(3)); - let (ks0, ks1, ks2, ks3) = ni::encrypt_4blocks_128_aesni(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let ptr = data.as_mut_ptr().add(offset); - let p0 = _mm_loadu_si128(ptr.cast()); - let p1 = _mm_loadu_si128(ptr.add(16).cast()); - let p2 = _mm_loadu_si128(ptr.add(32).cast()); - let p3 = _mm_loadu_si128(ptr.add(48).cast()); - let c0 = _mm_xor_si128(p0, ks0); - let c1 = _mm_xor_si128(p1, ks1); - let c2 = _mm_xor_si128(p2, ks2); - let c3 = _mm_xor_si128(p3, ks3); - _mm_storeu_si128(ptr.cast(), c0); - _mm_storeu_si128(ptr.add(16).cast(), c1); - _mm_storeu_si128(ptr.add(32).cast(), c2); - _mm_storeu_si128(ptr.add(48).cast(), c3); - acc = super::polyval::x86_pclmul_aggregate_4blocks_be_xmm_inline(acc, h_powers_rev, c0, c1, c2, c3); - - ctr = ctr.wrapping_add(4); - offset = offset.strict_add(64); - } - - while offset < data.len() { - let mut counter_block = [0u8; 16]; - counter_block[..12].copy_from_slice(&iv_prefix); - counter_block[12..16].copy_from_slice(&ctr.to_be_bytes()); - - let mut keystream = counter_block; - ni::encrypt_block_128(ni_rk, &mut keystream); - - let remaining = data.len().strict_sub(offset); - if remaining >= BLOCK_SIZE { - let ks = u128::from_ne_bytes(keystream); - let mut d = [0u8; BLOCK_SIZE]; - d.copy_from_slice(&data[offset..offset.strict_add(BLOCK_SIZE)]); - let xored = u128::from_ne_bytes(d) ^ ks; - let ciphertext = xored.to_ne_bytes(); - data[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&ciphertext); - acc ^= u128::from_be_bytes(ciphertext); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); - offset = offset.strict_add(BLOCK_SIZE); - } else { - let mut block = [0u8; BLOCK_SIZE]; - let mut i = 0usize; - while i < remaining { - data[offset.strict_add(i)] ^= keystream[i]; - block[i] = data[offset.strict_add(i)]; - i = i.strict_add(1); - } - acc ^= u128::from_be_bytes(block); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); - offset = offset.strict_add(remaining); - } - ctr = ctr.wrapping_add(1); - } - - acc - } -} - -/// AES-128 CTR decryption fused with 256-bit VAES/VPCLMUL GHASH accumulation. -/// -/// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VAES + VPCLMULQDQ + -/// PCLMULQDQ + AES + SSE2 + SSSE3. -#[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] -#[target_feature(enable = "aes,sse2,ssse3,avx2,avx512f,avx512vl,vaes,vpclmulqdq,pclmulqdq")] -#[allow(dead_code)] -pub(crate) unsafe fn aes128_ctr32_decrypt_be_y256_ghash( - ek: &Aes128EncKey, - initial_counter: &[u8; BLOCK_SIZE], - data: &mut [u8], - mut acc: u128, - h_polyval: u128, - h_powers_rev: &[u128; 4], - h_powers_rev_8: &[u128; 8], -) -> u128 { - use core::arch::x86_64::*; - - // SAFETY: fused x86 VAES-128 AES-GCM opening because: - // 1. This function's caller guarantees all required x86 target features. - // 2. Ciphertext registers are folded into GHASH before plaintext is stored back. - // 3. All pointer arithmetic stays inside checked chunk bounds. - unsafe { - let ni_rk = match &ek.inner { - Key128Inner::X86AesNi(rk) => rk, - _ => { - acc = ghash_ciphertext_fallback(acc, h_polyval, data); - aes128_ctr32_encrypt_be(ek, initial_counter, data); - return acc; - } - }; - - let iv_prefix: [u8; 12] = { - let mut buf = [0u8; 12]; - buf.copy_from_slice(&initial_counter[..12]); - buf - }; - let iv_words = [ - u32::from_le_bytes([iv_prefix[0], iv_prefix[1], iv_prefix[2], iv_prefix[3]]), - u32::from_le_bytes([iv_prefix[4], iv_prefix[5], iv_prefix[6], iv_prefix[7]]), - u32::from_le_bytes([iv_prefix[8], iv_prefix[9], iv_prefix[10], iv_prefix[11]]), - ]; - let mut ctr = u32::from_be_bytes([ - initial_counter[12], - initial_counter[13], - initial_counter[14], - initial_counter[15], - ]); - let mut offset = 0usize; - - #[cfg(target_os = "linux")] - if data.len() >= 128 { - let mut state = x86_64_asm::AesGcmX86State::new(acc, ctr); - // SAFETY: external x86-64 VAES-256 AES-128-GCM open kernel because: - // 1. This target-feature function is only entered after VAES, VPCLMULQDQ, AVX2, AVX-512VL, AES-NI, - // PCLMULQDQ, SSE2, and SSSE3 were selected by runtime/backend dispatch. - // 2. `ni_rk.as_ptr()` addresses 11 initialized 128-bit AES-128 round keys. - // 3. `initial_counter` points to the full 16-byte GCM counter block, and `data` is valid for - // `data.len()` mutable ciphertext bytes. - // 4. The kernel folds ciphertext into GHASH before storing plaintext. - // 5. The kernel only processes complete 128-byte chunks and reports the processed byte count so the - // Rust fallback below handles every remaining full/partial tail. - x86_64_asm::rscrypto_aes128_gcm_open_8x_vaes256_x86_64_linux( - ni_rk.as_ptr(), - initial_counter.as_ptr(), - data.as_mut_ptr(), - data.len(), - h_powers_rev_8.as_ptr(), - &mut state, - ); + } acc = state.acc(); ctr = state.ctr; offset = state.processed; } - while offset.strict_add(128) <= data.len() { - let (ctr0, ctr1, ctr2, ctr3) = x86_gcm_ctr_blocks_be_8_y256(iv_words, ctr); - let (ks0, ks1, ks2, ks3) = ni::encrypt_8blocks_128_y256(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let c0 = _mm256_loadu_si256(data.as_ptr().add(offset).cast()); - let c1 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(32)).cast()); - let c2 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(64)).cast()); - let c3 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(96)).cast()); - acc = super::polyval::x86_aggregate_8blocks_be_lanes_256_inline(acc, h_powers_rev_8, c0, c1, c2, c3); - - _mm256_storeu_si256(data.as_mut_ptr().add(offset).cast(), _mm256_xor_si256(c0, ks0)); - _mm256_storeu_si256( - data.as_mut_ptr().add(offset.strict_add(32)).cast(), - _mm256_xor_si256(c1, ks1), - ); - _mm256_storeu_si256( + while offset.strict_add(256) <= data.len() { + let (ctr0, ctr1, ctr2, ctr3) = x86_gcm_ctr_blocks_be_16(iv_words, ctr); + let (ks0, ks1, ks2, ks3) = ni::encrypt_16blocks_128(ni_rk, ctr0, ctr1, ctr2, ctr3); + + let c0 = _mm512_loadu_si512(data.as_ptr().add(offset).cast()); + let c1 = _mm512_loadu_si512(data.as_ptr().add(offset.strict_add(64)).cast()); + let c2 = _mm512_loadu_si512(data.as_ptr().add(offset.strict_add(128)).cast()); + let c3 = _mm512_loadu_si512(data.as_ptr().add(offset.strict_add(192)).cast()); + acc = super::polyval::x86_aggregate_16blocks_be_lanes_inline(acc, h_powers_rev_16, c0, c1, c2, c3); + _mm512_storeu_si512(data.as_mut_ptr().add(offset).cast(), _mm512_xor_si512(c0, ks0)); + _mm512_storeu_si512( data.as_mut_ptr().add(offset.strict_add(64)).cast(), - _mm256_xor_si256(c2, ks2), + _mm512_xor_si512(c1, ks1), + ); + _mm512_storeu_si512( + data.as_mut_ptr().add(offset.strict_add(128)).cast(), + _mm512_xor_si512(c2, ks2), ); - _mm256_storeu_si256( - data.as_mut_ptr().add(offset.strict_add(96)).cast(), - _mm256_xor_si256(c3, ks3), + _mm512_storeu_si512( + data.as_mut_ptr().add(offset.strict_add(192)).cast(), + _mm512_xor_si512(c3, ks3), ); - ctr = ctr.wrapping_add(8); - offset = offset.strict_add(128); + ctr = ctr.wrapping_add(16); + offset = offset.strict_add(256); } while offset.strict_add(64) <= data.len() { - let ctr0 = x86_gcm_ctr_block_be(iv_words, ctr); - let ctr1 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(1)); - let ctr2 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(2)); - let ctr3 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(3)); - let (ks0, ks1, ks2, ks3) = ni::encrypt_4blocks_128_aesni(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let ptr = data.as_mut_ptr().add(offset); - let c0 = _mm_loadu_si128(ptr.cast()); - let c1 = _mm_loadu_si128(ptr.add(16).cast()); - let c2 = _mm_loadu_si128(ptr.add(32).cast()); - let c3 = _mm_loadu_si128(ptr.add(48).cast()); - acc = super::polyval::x86_pclmul_aggregate_4blocks_be_xmm_inline(acc, h_powers_rev, c0, c1, c2, c3); - _mm_storeu_si128(ptr.cast(), _mm_xor_si128(c0, ks0)); - _mm_storeu_si128(ptr.add(16).cast(), _mm_xor_si128(c1, ks1)); - _mm_storeu_si128(ptr.add(32).cast(), _mm_xor_si128(c2, ks2)); - _mm_storeu_si128(ptr.add(48).cast(), _mm_xor_si128(c3, ks3)); + let ctr_vec = x86_gcm_ctr_blocks_be_4(iv_words, ctr); + let keystream = ni::encrypt_4blocks_128(ni_rk, ctr_vec); + let ciphertext = _mm512_loadu_si512(data.as_ptr().add(offset).cast()); + acc = super::polyval::x86_aggregate_4blocks_be_lanes_inline(acc, h_powers_rev, ciphertext); + _mm512_storeu_si512( + data.as_mut_ptr().add(offset).cast(), + _mm512_xor_si512(ciphertext, keystream), + ); ctr = ctr.wrapping_add(4); offset = offset.strict_add(64); @@ -5251,9 +4480,19 @@ pub(crate) unsafe fn aes128_ctr32_decrypt_be_y256_ghash( } } +#[cfg(all(target_arch = "x86_64", target_os = "linux", feature = "aes-gcm-siv"))] +#[inline(always)] +const fn usize_low_u32(value: usize) -> u32 { + let bytes = value.to_le_bytes(); + u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) +} + /// AES-256 CTR encryption using VAES-512 for the bulk, AES-NI for the tail. /// /// GCM-SIV variant: counter occupies bytes 0..3 (little-endian). +/// +/// # Safety +/// Caller must ensure AES-NI, SSE2, AVX-512F, AVX-512VL, and VAES are available. #[cfg(all(target_arch = "x86_64", feature = "aes-gcm-siv"))] #[target_feature(enable = "aes,sse2,avx512f,avx512vl,vaes")] pub(crate) unsafe fn aes256_ctr32_encrypt_wide(ek: &Aes256EncKey, initial_counter: &[u8; BLOCK_SIZE], data: &mut [u8]) { @@ -5304,7 +4543,7 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_wide(ek: &Aes256EncKey, initial_counte data.as_mut_ptr(), data.len(), ); - ctr = ctr.wrapping_add((processed / BLOCK_SIZE) as u32); + ctr = ctr.wrapping_add(usize_low_u32(processed.strict_div(BLOCK_SIZE))); offset = processed; } @@ -5439,7 +4678,7 @@ pub(crate) unsafe fn aes128_ctr32_encrypt_wide(ek: &Aes128EncKey, initial_counte data.as_mut_ptr(), data.len(), ); - ctr = ctr.wrapping_add((processed / BLOCK_SIZE) as u32); + ctr = ctr.wrapping_add(usize_low_u32(processed.strict_div(BLOCK_SIZE))); offset = processed; } @@ -5544,12 +4783,12 @@ mod tests { 0x8c,0xa1,0x89,0x0d,0xbf,0xe6,0x42,0x68,0x41,0x99,0x2d,0x0f,0xb0,0x54,0xbb,0x16, ]; - for (i, &expected) in CANONICAL.iter().enumerate() { + for (input, &expected) in (0u8..=u8::MAX).zip(&CANONICAL) { assert_eq!( - sbox(i as u8), + sbox(input), expected, - "S-box mismatch at index {i:#04x}: got {:#04x}, expected {expected:#04x}", - sbox(i as u8), + "S-box mismatch at input {input:#04x}: got {:#04x}, expected {expected:#04x}", + sbox(input), ); } } @@ -5644,13 +4883,44 @@ mod tests { #[test] fn gf256_inv_exhaustive() { assert_eq!(gf256_inv(0), 0, "inv(0) must be 0 by AES convention"); - for x in 1u16..=255 { - let x = x as u8; + for x in 1u8..=u8::MAX { let inv = gf256_inv(x); assert_eq!(gf256_mul(x, inv), 1, "x={x:#04x}, inv={inv:#04x}: x * inv != 1"); } } + #[cfg(all(target_arch = "aarch64", feature = "aes-gcm-siv"))] + #[test] + fn aarch64_gcmsiv_kdf_portable_fallback_matches_aes_ce() { + if !crate::platform::caps().has(crate::platform::caps::aarch64::AES) { + return; + } + + let nonce = *b"kdf nonce 12"; + let key_128 = [0x39; KEY_SIZE_128]; + let key_256 = [0xa7; KEY_SIZE]; + let accelerated_128 = aes128_expand_key(&key_128); + let accelerated_256 = aes256_expand_key(&key_256); + let portable_128 = Aes128EncKey { + inner: Key128Inner::PortableRoundKeys(aes128_expand_key_portable(&key_128)), + }; + let portable_256 = Aes256EncKey { + inner: KeyInner::PortableRoundKeys(aes256_expand_key_portable(&key_256)), + }; + + // SAFETY: runtime capabilities above confirm AES-CE before both target-feature calls. + let expected_128 = unsafe { aarch64_gcmsiv_derive_keys_128_inline(&accelerated_128, &nonce) }; + // SAFETY: runtime capabilities above confirm AES-CE; the portable key exercises the safe fallback. + let actual_128 = unsafe { aarch64_gcmsiv_derive_keys_128_inline(&portable_128, &nonce) }; + assert_eq!(actual_128, expected_128); + + // SAFETY: runtime capabilities above confirm AES-CE before both target-feature calls. + let expected_256 = unsafe { aarch64_gcmsiv_derive_keys_inline(&accelerated_256, &nonce) }; + // SAFETY: runtime capabilities above confirm AES-CE; the portable key exercises the safe fallback. + let actual_256 = unsafe { aarch64_gcmsiv_derive_keys_inline(&portable_256, &nonce) }; + assert_eq!(actual_256, expected_256); + } + /// AES-256 CTR mode: round-trip (encrypt then decrypt = identity). #[cfg(feature = "aes-gcm-siv")] #[test] @@ -5713,18 +4983,12 @@ mod tests { let mut buf = [0u8; 80]; aes128_ctr32_encrypt_be(&ek, &iv, &mut buf); - for block_idx in 0..5usize { + for (block_idx, actual) in (0u32..5).zip(buf.as_chunks::().0) { let mut expected = iv; - let ctr = 3u32.wrapping_add(block_idx as u32); + let ctr = 3u32.wrapping_add(block_idx); expected[12..16].copy_from_slice(&ctr.to_be_bytes()); aes128_encrypt_block(&ek, &mut expected); - let start = block_idx.strict_mul(BLOCK_SIZE); - let end = start.strict_add(BLOCK_SIZE); - assert_eq!( - &buf[start..end], - &expected, - "AES-128 CTR-BE block {block_idx} keystream mismatch" - ); + assert_eq!(actual, &expected, "AES-128 CTR-BE block {block_idx} keystream mismatch"); } } @@ -5741,16 +5005,13 @@ mod tests { let mut buf = [0u8; 80]; aes128_ctr32_encrypt_be(&ek, &iv, &mut buf); - for block_idx in 0..5usize { + for (block_idx, actual) in (0u32..5).zip(buf.as_chunks::().0) { let mut expected = iv; - let ctr = start_ctr.wrapping_add(block_idx as u32); + let ctr = start_ctr.wrapping_add(block_idx); expected[12..16].copy_from_slice(&ctr.to_be_bytes()); aes128_encrypt_block(&ek, &mut expected); - let start = block_idx.strict_mul(BLOCK_SIZE); - let end = start.strict_add(BLOCK_SIZE); assert_eq!( - &buf[start..end], - &expected, + actual, &expected, "AES-128 CTR-BE wrap block {block_idx} keystream mismatch" ); } @@ -5768,18 +5029,12 @@ mod tests { let mut buf = [0u8; 80]; aes256_ctr32_encrypt_be(&ek, &iv, &mut buf); - for block_idx in 0..5usize { + for (block_idx, actual) in (0u32..5).zip(buf.as_chunks::().0) { let mut expected = iv; - let ctr = 7u32.wrapping_add(block_idx as u32); + let ctr = 7u32.wrapping_add(block_idx); expected[12..16].copy_from_slice(&ctr.to_be_bytes()); aes256_encrypt_block(&ek, &mut expected); - let start = block_idx.strict_mul(BLOCK_SIZE); - let end = start.strict_add(BLOCK_SIZE); - assert_eq!( - &buf[start..end], - &expected, - "CTR-BE block {block_idx} keystream mismatch" - ); + assert_eq!(actual, &expected, "CTR-BE block {block_idx} keystream mismatch"); } } @@ -5796,16 +5051,13 @@ mod tests { let mut buf = [0u8; 80]; aes256_ctr32_encrypt_be(&ek, &iv, &mut buf); - for block_idx in 0..5usize { + for (block_idx, actual) in (0u32..5).zip(buf.as_chunks::().0) { let mut expected = iv; - let ctr = start_ctr.wrapping_add(block_idx as u32); + let ctr = start_ctr.wrapping_add(block_idx); expected[12..16].copy_from_slice(&ctr.to_be_bytes()); aes256_encrypt_block(&ek, &mut expected); - let start = block_idx.strict_mul(BLOCK_SIZE); - let end = start.strict_add(BLOCK_SIZE); assert_eq!( - &buf[start..end], - &expected, + actual, &expected, "AES-256 CTR-BE wrap block {block_idx} keystream mismatch" ); } @@ -5891,7 +5143,9 @@ mod tests { fn fill_aarch64_gcm_test_plaintext(out: &mut [u8; N]) { let mut i = 0usize; while i < N { - out[i] = (i as u8).wrapping_mul(0x3d).wrapping_add(0x47) ^ ((i >> 3) as u8).wrapping_mul(0x91); + let index = i.to_le_bytes()[0]; + let group = (i >> 3).to_le_bytes()[0]; + out[i] = index.wrapping_mul(0x3d).wrapping_add(0x47) ^ group.wrapping_mul(0x91); i = i.strict_add(1); } } @@ -6146,29 +5400,6 @@ mod tests { out } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[target_feature(enable = "avx2")] - /// # Safety - /// - /// Caller must ensure AVX2 is available before calling this target-feature helper. - unsafe fn x86_gcm_ctr_blocks_be_8_y256_test_bytes(iv_words: [u32; 3], ctr: u32) -> [u8; 128] { - use core::arch::x86_64::*; - - let mut out = [0u8; 128]; - // SAFETY: test-only YMM counter block stores because: - // 1. The caller verified AVX2 before invoking this target-feature helper. - // 2. `out` is exactly 128 writable bytes, matching four contiguous `__m256i` stores. - // 3. The vectors under test are produced directly from the eight-block counter constructor. - unsafe { - let (c0, c1, c2, c3) = x86_gcm_ctr_blocks_be_8_y256(iv_words, ctr); - _mm256_storeu_si256(out.as_mut_ptr().cast(), c0); - _mm256_storeu_si256(out.as_mut_ptr().add(32).cast(), c1); - _mm256_storeu_si256(out.as_mut_ptr().add(64).cast(), c2); - _mm256_storeu_si256(out.as_mut_ptr().add(96).cast(), c3); - } - out - } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] #[test] fn x86_gcm_ctr_block_be_preserves_prefix_and_encodes_counter() { @@ -6263,38 +5494,6 @@ mod tests { } } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[test] - fn x86_gcm_ctr_blocks_be_8_y256_preserves_prefix_and_wraps_counter() { - if !crate::platform::caps().has(crate::platform::caps::x86::AVX2) { - return; - } - - let iv_prefix = *b"ctr wrap iv!"; - let iv_words = x86_gcm_iv_words(&iv_prefix); - - for ctr in [0x0102_0304, u32::MAX - 3] { - let mut expected = [0u8; 128]; - fill_expected_gcm_counter_blocks(&iv_prefix, ctr, &mut expected); - // SAFETY: runtime caps above confirmed AVX2 before calling the target-feature helper. - let actual = unsafe { x86_gcm_ctr_blocks_be_8_y256_test_bytes(iv_words, ctr) }; - assert_eq!( - actual.as_slice(), - expected.as_slice(), - "x86 vectorized YMM GCM counter block mismatch" - ); - } - } - - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - fn x86_y256_gcm_caps_available() -> bool { - let required = crate::platform::caps::x86::VAES_READY - | crate::platform::caps::x86::VPCLMUL_READY - | crate::platform::caps::x86::AVX2 - | crate::platform::caps::x86::AESNI; - crate::platform::caps().has(required) - } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] fn x86_z512_gcm_caps_available() -> bool { let required = crate::platform::caps::x86::VAES_READY @@ -6303,19 +5502,6 @@ mod tests { crate::platform::caps().has(required) } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - fn x86_gcm_test_powers() -> (u128, [u128; 4], [u128; 8]) { - let h_polyval = 0x1287_3d5b_fedc_ba09_7654_3210_f0e1_d2c3u128; - let powers = crate::aead::polyval::precompute_powers_8(h_polyval); - ( - h_polyval, - [powers[3], powers[2], powers[1], powers[0]], - [ - powers[7], powers[6], powers[5], powers[4], powers[3], powers[2], powers[1], powers[0], - ], - ) - } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] struct X86GcmTestPowers16 { h_polyval: u128, @@ -6383,327 +5569,13 @@ mod tests { fn fill_x86_gcm_test_plaintext(out: &mut [u8]) { let mut i = 0usize; while i < out.len() { - out[i] = (i as u8).wrapping_mul(0x3d).wrapping_add(0x47) ^ ((i >> 3) as u8).wrapping_mul(0x91); + let index = i.to_le_bytes()[0]; + let group = (i >> 3).to_le_bytes()[0]; + out[i] = index.wrapping_mul(0x3d).wrapping_add(0x47) ^ group.wrapping_mul(0x91); i = i.strict_add(1); } } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[test] - fn x86_aes128_gcm_y256_encrypt_matches_scalar_across_counter_wrap() { - if !x86_y256_gcm_caps_available() { - return; - } - - let ek = aes128_expand_key(&[0xA1u8; KEY_SIZE_128]); - let counter = x86_gcm_wrap_counter_block(); - let (h_polyval, h_powers_rev, h_powers_rev_8) = x86_gcm_test_powers(); - let seed_acc = 0xfeed_face_cafe_babe_1020_3040_5060_7080u128; - - let mut plaintext = [0u8; 128]; - fill_x86_gcm_test_plaintext(&mut plaintext); - let mut expected = plaintext; - aes128_ctr32_encrypt_be(&ek, &counter, &mut expected); - let expected_acc = ghash_ciphertext_fallback(seed_acc, h_polyval, &expected); - - let mut actual = plaintext; - // SAFETY: runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI before calling - // the y256 target-feature helper. Inputs are fixed-size initialized test buffers. - let actual_acc = unsafe { - aes128_ctr32_encrypt_be_y256_ghash( - &ek, - &counter, - &mut actual, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!( - actual, expected, - "AES-128 y256 seal ciphertext must match scalar CTR across wrap" - ); - assert_eq!( - actual_acc, expected_acc, - "AES-128 y256 seal GHASH accumulator must match scalar fold across wrap" - ); - } - - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[test] - fn x86_aes128_gcm_y256_decrypt_matches_scalar_across_counter_wrap() { - if !x86_y256_gcm_caps_available() { - return; - } - - let ek = aes128_expand_key(&[0xB2u8; KEY_SIZE_128]); - let counter = x86_gcm_wrap_counter_block(); - let (h_polyval, h_powers_rev, h_powers_rev_8) = x86_gcm_test_powers(); - let seed_acc = 0x9ace_0246_8bdf_1357_1122_3344_5566_7788u128; - - let mut plaintext = [0u8; 128]; - fill_x86_gcm_test_plaintext(&mut plaintext); - let mut ciphertext = plaintext; - aes128_ctr32_encrypt_be(&ek, &counter, &mut ciphertext); - let expected_acc = ghash_ciphertext_fallback(seed_acc, h_polyval, &ciphertext); - - let mut actual = ciphertext; - // SAFETY: runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI before calling - // the y256 target-feature helper. Inputs are fixed-size initialized test buffers. - let actual_acc = unsafe { - aes128_ctr32_decrypt_be_y256_ghash( - &ek, - &counter, - &mut actual, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!( - actual, plaintext, - "AES-128 y256 open plaintext must match scalar CTR across wrap" - ); - assert_eq!( - actual_acc, expected_acc, - "AES-128 y256 open GHASH accumulator must match scalar fold across wrap" - ); - } - - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[test] - fn x86_aes256_gcm_y256_encrypt_matches_scalar_across_counter_wrap() { - if !x86_y256_gcm_caps_available() { - return; - } - - let ek = aes256_expand_key(&[0xC3u8; KEY_SIZE]); - let counter = x86_gcm_wrap_counter_block(); - let (h_polyval, h_powers_rev, h_powers_rev_8) = x86_gcm_test_powers(); - let seed_acc = 0x0123_4567_89ab_cdef_fedc_ba98_7654_3210u128; - - let mut plaintext = [0u8; 128]; - fill_x86_gcm_test_plaintext(&mut plaintext); - let mut expected = plaintext; - aes256_ctr32_encrypt_be(&ek, &counter, &mut expected); - let expected_acc = ghash_ciphertext_fallback(seed_acc, h_polyval, &expected); - - let mut actual = plaintext; - // SAFETY: runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI before calling - // the y256 target-feature helper. Inputs are fixed-size initialized test buffers. - let actual_acc = unsafe { - aes256_ctr32_encrypt_be_y256_ghash( - &ek, - &counter, - &mut actual, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!( - actual, expected, - "AES-256 y256 seal ciphertext must match scalar CTR across wrap" - ); - assert_eq!( - actual_acc, expected_acc, - "AES-256 y256 seal GHASH accumulator must match scalar fold across wrap" - ); - } - - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[test] - fn x86_aes256_gcm_y256_decrypt_matches_scalar_across_counter_wrap() { - if !x86_y256_gcm_caps_available() { - return; - } - - let ek = aes256_expand_key(&[0xD4u8; KEY_SIZE]); - let counter = x86_gcm_wrap_counter_block(); - let (h_polyval, h_powers_rev, h_powers_rev_8) = x86_gcm_test_powers(); - let seed_acc = 0xaa55_aa55_55aa_55aa_cc33_cc33_33cc_33ccu128; - - let mut plaintext = [0u8; 128]; - fill_x86_gcm_test_plaintext(&mut plaintext); - let mut ciphertext = plaintext; - aes256_ctr32_encrypt_be(&ek, &counter, &mut ciphertext); - let expected_acc = ghash_ciphertext_fallback(seed_acc, h_polyval, &ciphertext); - - let mut actual = ciphertext; - // SAFETY: runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI before calling - // the y256 target-feature helper. Inputs are fixed-size initialized test buffers. - let actual_acc = unsafe { - aes256_ctr32_decrypt_be_y256_ghash( - &ek, - &counter, - &mut actual, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!( - actual, plaintext, - "AES-256 y256 open plaintext must match scalar CTR across wrap" - ); - assert_eq!( - actual_acc, expected_acc, - "AES-256 y256 open GHASH accumulator must match scalar fold across wrap" - ); - } - - #[cfg(all( - target_arch = "x86_64", - feature = "aes-gcm", - any(target_os = "linux", target_os = "macos", target_os = "windows") - ))] - #[test] - fn x86_aes128_gcm_y256_large_asm_tail_matches_scalar() { - if !x86_y256_gcm_caps_available() { - return; - } - - const LEN: usize = 1057; - - let ek = aes128_expand_key(&[0xE5u8; KEY_SIZE_128]); - let counter = x86_gcm_wrap_counter_block(); - let (h_polyval, h_powers_rev, h_powers_rev_8) = x86_gcm_test_powers(); - let seed_acc = 0x3141_5926_5358_9793_2384_6264_3383_2795u128; - - let mut plaintext = [0u8; LEN]; - fill_x86_gcm_test_plaintext(&mut plaintext); - - let mut expected = plaintext; - aes128_ctr32_encrypt_be(&ek, &counter, &mut expected); - let expected_acc = ghash_ciphertext_fallback(seed_acc, h_polyval, &expected); - - let mut actual = plaintext; - // SAFETY: large x86 AES-128-GCM seal test because: - // 1. Runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI. - // 2. The 1057-byte input forces the x86 ASM bulk path to process full blocks. - // 3. The final byte is handled by the shared Rust tail after the ASM path returns. - let actual_acc = unsafe { - aes128_ctr32_encrypt_be_y256_ghash( - &ek, - &counter, - &mut actual, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!(actual, expected, "AES-128 large y256/ASM seal ciphertext mismatch"); - assert_eq!( - actual_acc, expected_acc, - "AES-128 large y256/ASM seal GHASH accumulator mismatch" - ); - - let mut opened = actual; - // SAFETY: large x86 AES-128-GCM open test because: - // 1. Runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI. - // 2. The 1057-byte input forces the x86 ASM bulk path to process full blocks. - // 3. The helper GHASHes ciphertext before decrypting and leaves the final byte to the Rust tail. - let open_acc = unsafe { - aes128_ctr32_decrypt_be_y256_ghash( - &ek, - &counter, - &mut opened, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!(opened, plaintext, "AES-128 large y256/ASM open plaintext mismatch"); - assert_eq!( - open_acc, expected_acc, - "AES-128 large y256/ASM open GHASH accumulator mismatch" - ); - } - - #[cfg(all( - target_arch = "x86_64", - feature = "aes-gcm", - any(target_os = "linux", target_os = "macos", target_os = "windows") - ))] - #[test] - fn x86_aes256_gcm_y256_large_asm_tail_matches_scalar() { - if !x86_y256_gcm_caps_available() { - return; - } - - const LEN: usize = 1057; - - let ek = aes256_expand_key(&[0xF6u8; KEY_SIZE]); - let counter = x86_gcm_wrap_counter_block(); - let (h_polyval, h_powers_rev, h_powers_rev_8) = x86_gcm_test_powers(); - let seed_acc = 0x2718_2818_2845_9045_2353_6028_7471_3526u128; - - let mut plaintext = [0u8; LEN]; - fill_x86_gcm_test_plaintext(&mut plaintext); - - let mut expected = plaintext; - aes256_ctr32_encrypt_be(&ek, &counter, &mut expected); - let expected_acc = ghash_ciphertext_fallback(seed_acc, h_polyval, &expected); - - let mut actual = plaintext; - // SAFETY: large x86 AES-256-GCM seal test because: - // 1. Runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI. - // 2. The 1057-byte input forces the x86 ASM bulk path to process full blocks. - // 3. The final byte is handled by the shared Rust tail after the ASM path returns. - let actual_acc = unsafe { - aes256_ctr32_encrypt_be_y256_ghash( - &ek, - &counter, - &mut actual, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!(actual, expected, "AES-256 large y256/ASM seal ciphertext mismatch"); - assert_eq!( - actual_acc, expected_acc, - "AES-256 large y256/ASM seal GHASH accumulator mismatch" - ); - - let mut opened = actual; - // SAFETY: large x86 AES-256-GCM open test because: - // 1. Runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI. - // 2. The 1057-byte input forces the x86 ASM bulk path to process full blocks. - // 3. The helper GHASHes ciphertext before decrypting and leaves the final byte to the Rust tail. - let open_acc = unsafe { - aes256_ctr32_decrypt_be_y256_ghash( - &ek, - &counter, - &mut opened, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!(opened, plaintext, "AES-256 large y256/ASM open plaintext mismatch"); - assert_eq!( - open_acc, expected_acc, - "AES-256 large y256/ASM open GHASH accumulator mismatch" - ); - } - #[cfg(all( target_arch = "x86_64", feature = "aes-gcm", @@ -6834,9 +5706,9 @@ mod tests { let ek = aes128_expand_key(&key); let mut blocks = [[0u8; BLOCK_SIZE]; 6]; - for (i, block) in blocks.iter_mut().enumerate() { - for (j, byte) in block.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(13) ^ (j as u8).wrapping_mul(31) ^ 0xA3; + for (i, block) in (0u8..).zip(&mut blocks) { + for (j, byte) in (0u8..).zip(block) { + *byte = i.wrapping_mul(13) ^ j.wrapping_mul(31) ^ 0xA3; } } @@ -6855,9 +5727,9 @@ mod tests { let ek = aes256_expand_key(&key); let mut blocks = [[0u8; BLOCK_SIZE]; 6]; - for (i, block) in blocks.iter_mut().enumerate() { - for (j, byte) in block.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(17) ^ (j as u8).wrapping_mul(29) ^ 0x5C; + for (i, block) in (0u8..).zip(&mut blocks) { + for (j, byte) in (0u8..).zip(block) { + *byte = i.wrapping_mul(17) ^ j.wrapping_mul(29) ^ 0x5C; } } @@ -6896,9 +5768,9 @@ mod tests { let fixslice = rv_fixslice_aes::RvFixslice128RoundKeys::new(&key); let mut blocks = [[0u8; BLOCK_SIZE]; 4]; - for (i, block) in blocks.iter_mut().enumerate() { - for (j, byte) in block.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(0x47) ^ (j as u8).wrapping_mul(0x6d) ^ 0x9c; + for (i, block) in (0u8..).zip(&mut blocks) { + for (j, byte) in (0u8..).zip(block) { + *byte = i.wrapping_mul(0x47) ^ j.wrapping_mul(0x6d) ^ 0x9c; } } @@ -6937,9 +5809,9 @@ mod tests { let fixslice = rv_fixslice_aes::RvFixsliceRoundKeys::new(&key); let mut blocks = [[0u8; BLOCK_SIZE]; 4]; - for (i, block) in blocks.iter_mut().enumerate() { - for (j, byte) in block.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(0x31) ^ (j as u8).wrapping_mul(0x57) ^ 0xa6; + for (i, block) in (0u8..).zip(&mut blocks) { + for (j, byte) in (0u8..).zip(block) { + *byte = i.wrapping_mul(0x31) ^ j.wrapping_mul(0x57) ^ 0xa6; } } diff --git a/src/aead/aes/aarch64/asm.rs b/src/aead/aes/aarch64/asm.rs index f5115eb7..e40c9902 100644 --- a/src/aead/aes/aarch64/asm.rs +++ b/src/aead/aes/aarch64/asm.rs @@ -1,7 +1,5 @@ //! AArch64 rscrypto-owned AES-GCM assembly kernels. -#![allow(unsafe_code)] - use core::arch::global_asm; global_asm!(include_str!("asm/rscrypto_aes_gcm_aarch64_apple_darwin.s")); @@ -19,8 +17,8 @@ impl AesGcmAarch64State { #[inline] pub(super) fn new(acc: u128, ctr: u32) -> Self { Self { - acc_lo: acc as u64, - acc_hi: (acc >> 64) as u64, + acc_lo: u64::try_from(acc & u128::from(u64::MAX)).expect("masked accumulator half fits u64"), + acc_hi: u64::try_from(acc >> 64).expect("shifted accumulator half fits u64"), ctr, _pad: 0, processed: 0, diff --git a/src/aead/aes/aarch64_ce.rs b/src/aead/aes/aarch64_ce.rs index b22c1cd5..ea1250aa 100644 --- a/src/aead/aes/aarch64_ce.rs +++ b/src/aead/aes/aarch64_ce.rs @@ -266,6 +266,11 @@ pub(super) unsafe fn encrypt_6blocks_core(keys: &CeRoundKeys, blocks: &mut [[u8; #[cfg(feature = "aes-gcm")] #[inline] +/// Convert one initialized GHASH byte vector to its big-endian `u128` value. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available. unsafe fn ghash_be_u128_from_vec(block: uint8x16_t) -> u128 { // SAFETY: GHASH lane conversion because: // 1. The caller is already inside an AArch64 NEON target scope. @@ -282,16 +287,48 @@ unsafe fn ghash_be_u128_from_vec(block: uint8x16_t) -> u128 { #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Split one GHASH accumulator into its low and high 64-bit NEON lanes. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available. unsafe fn ghash_u128_to_lanes(x: u128) -> uint64x2_t { + let [ + lo_0, + lo_1, + lo_2, + lo_3, + lo_4, + lo_5, + lo_6, + lo_7, + hi_0, + hi_1, + hi_2, + hi_3, + hi_4, + hi_5, + hi_6, + hi_7, + ] = x.to_le_bytes(); + let lo = u64::from_le_bytes([lo_0, lo_1, lo_2, lo_3, lo_4, lo_5, lo_6, lo_7]); + let hi = u64::from_le_bytes([hi_0, hi_1, hi_2, hi_3, hi_4, hi_5, hi_6, hi_7]); + // SAFETY: GHASH accumulator lane construction because: // 1. The caller is already inside an AArch64 NEON target scope. // 2. `vcreate_u64` initializes one 64-bit lane from an integer value. // 3. `vcombine_u64` builds a fully initialized two-lane vector. - unsafe { vcombine_u64(vcreate_u64(x as u64), vcreate_u64((x >> 64) as u64)) } + unsafe { vcombine_u64(vcreate_u64(lo), vcreate_u64(hi)) } } #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Reduce one 256-bit GHASH carryless product to two Montgomery-form lanes. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available and pass initialized low and high +/// product halves. unsafe fn ghash_mont_reduce_neon(lo: uint64x2_t, hi: uint64x2_t) -> uint64x2_t { // SAFETY: GHASH Montgomery reduction because: // 1. The caller is already inside an AArch64 NEON target scope. @@ -315,6 +352,12 @@ unsafe fn ghash_mont_reduce_neon(lo: uint64x2_t, hi: uint64x2_t) -> uint64x2_t { #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Finish the three Karatsuba accumulators and return the reduced GHASH value. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available and pass initialized low, high, and +/// middle product accumulators for the same fold. unsafe fn ghash_finish_products(ll: uint64x2_t, hh: uint64x2_t, mm: uint64x2_t) -> u128 { // SAFETY: GHASH Karatsuba accumulator finalization because: // 1. The caller is already inside an AArch64 NEON target scope. @@ -332,6 +375,12 @@ unsafe fn ghash_finish_products(ll: uint64x2_t, hh: uint64x2_t, mm: uint64x2_t) #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Load one ciphertext block without imposing an alignment requirement. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available. `ptr` must remain valid and readable +/// for 16 initialized bytes for the duration of the call; it need not be aligned. unsafe fn gcm_load_ciphertext_block(ptr: *const u8) -> uint8x16_t { // SAFETY: opaque 16-byte ciphertext load for GCM open because: // 1. The caller passes a pointer into a 128-byte chunk already bounds-checked by the enclosing @@ -353,6 +402,12 @@ unsafe fn gcm_load_ciphertext_block(ptr: *const u8) -> uint8x16_t { #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Load one GHASH key power as two unaligned 64-bit lanes. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available. `power` must remain valid and readable +/// for one initialized `u128` for the duration of the call; it need not be aligned. unsafe fn ghash_load_power(power: *const u128) -> uint64x2_t { // SAFETY: GHASH H-power vector load because: // 1. The caller passes a pointer into a live H-power table. @@ -395,6 +450,11 @@ macro_rules! gcm_schedule_barrier { #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Build the fixed 96-bit prefix of an AES-GCM big-endian counter block. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available. unsafe fn gcm_ctr32_base(iv_prefix: &[u8; 12]) -> uint8x16_t { let mut block = [0u8; 16]; block[..12].copy_from_slice(iv_prefix); @@ -407,6 +467,11 @@ unsafe fn gcm_ctr32_base(iv_prefix: &[u8; 12]) -> uint8x16_t { #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Insert one AES-GCM counter into the big-endian final lane. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available and pass an initialized counter base. unsafe fn gcm_ctr32_block(base: uint8x16_t, ctr: u32) -> uint8x16_t { // SAFETY: GCM counter lane update because: // 1. The caller is already inside an AArch64 NEON target scope. @@ -417,6 +482,11 @@ unsafe fn gcm_ctr32_block(base: uint8x16_t, ctr: u32) -> uint8x16_t { #[cfg(feature = "aes-gcm-siv")] #[inline(always)] +/// Build the fixed 96-bit suffix of an AES-GCM-SIV little-endian counter block. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available. unsafe fn gcmsiv_ctr32_base(iv_suffix: &[u8; 12]) -> uint8x16_t { let mut block = [0u8; 16]; block[4..16].copy_from_slice(iv_suffix); @@ -429,6 +499,11 @@ unsafe fn gcmsiv_ctr32_base(iv_suffix: &[u8; 12]) -> uint8x16_t { #[cfg(feature = "aes-gcm-siv")] #[inline(always)] +/// Insert one AES-GCM-SIV counter into the little-endian first lane. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available and pass an initialized counter base. unsafe fn gcmsiv_ctr32_block(base: uint8x16_t, ctr: u32) -> uint8x16_t { // SAFETY: GCM-SIV counter lane update because: // 1. The caller is inside a NEON target scope. @@ -751,6 +826,12 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_8blocks_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline] +/// Encrypt eight AES-256-GCM blocks while folding the preceding ciphertext group. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available; `data` contains at least 128 bytes; and the round keys, +/// accumulator, GHASH powers, and preceding ciphertext lanes belong to the same GCM key and stream. pub(super) unsafe fn encrypt_ctr32_be_xor_8blocks_ghash_prev_bytes_core( keys: &CeRoundKeys, iv_prefix: &[u8; 12], @@ -883,6 +964,12 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_8blocks_ghash_prev_bytes_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline] +/// Fold and decrypt eight current AES-256-GCM ciphertext blocks. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available; `data` contains at least 128 bytes; and the round keys, +/// accumulator, and GHASH powers belong to the same GCM key and stream. pub(super) unsafe fn decrypt_ctr32_be_xor_8blocks_ghash_current_core( keys: &CeRoundKeys, iv_prefix: &[u8; 12], @@ -1071,6 +1158,12 @@ pub(super) unsafe fn decrypt_ctr32_be_xor_8blocks_ghash_current_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline(never)] +/// Encrypt complete 128-byte AES-256-GCM chunks and fold their ciphertext into GHASH. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available; `data` contains at least 128 bytes; and the round keys, +/// counter prefix, accumulator, and GHASH tables belong to the same GCM key and stream. pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_core( keys: &CeRoundKeys, iv_prefix: &[u8; 12], @@ -1102,7 +1195,7 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_core( tables.h_powers_rev_16.as_ptr(), tables.h_powers_rev_16_mid.as_ptr(), tables.h_powers_rev_16_pair.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -1116,7 +1209,7 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_core( data.as_mut_ptr(), data.len(), tables.h_powers_rev_8.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -1157,6 +1250,12 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline(never)] +/// Fold and decrypt complete 128-byte AES-256-GCM ciphertext chunks. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available and the round keys, counter prefix, accumulator, and +/// GHASH tables belong to the same GCM key and stream. `data` may be shorter than one complete chunk. pub(super) unsafe fn decrypt_ctr32_be_xor_ghash_128b_chunks_core( keys: &CeRoundKeys, iv_prefix: &[u8; 12], @@ -1184,7 +1283,7 @@ pub(super) unsafe fn decrypt_ctr32_be_xor_ghash_128b_chunks_core( tables.h_powers_rev_16.as_ptr(), tables.h_powers_rev_16_mid.as_ptr(), tables.h_powers_rev_16_pair.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -1198,7 +1297,7 @@ pub(super) unsafe fn decrypt_ctr32_be_xor_ghash_128b_chunks_core( data.as_mut_ptr(), data.len(), tables.h_powers_rev_8.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -1733,6 +1832,12 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_8blocks_128_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline] +/// Encrypt eight AES-128-GCM blocks while folding the preceding ciphertext group. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available; `data` contains at least 128 bytes; and the round keys, +/// accumulator, GHASH powers, and preceding ciphertext lanes belong to the same GCM key and stream. pub(super) unsafe fn encrypt_ctr32_be_xor_8blocks_ghash_prev_bytes_128_core( keys: &Ce128RoundKeys, iv_prefix: &[u8; 12], @@ -1858,6 +1963,12 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_8blocks_ghash_prev_bytes_128_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline] +/// Fold and decrypt eight current AES-128-GCM ciphertext blocks. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available; `data` contains at least 128 bytes; and the round keys, +/// accumulator, and GHASH powers belong to the same GCM key and stream. pub(super) unsafe fn decrypt_ctr32_be_xor_8blocks_ghash_current_128_core( keys: &Ce128RoundKeys, iv_prefix: &[u8; 12], @@ -2039,6 +2150,12 @@ pub(super) unsafe fn decrypt_ctr32_be_xor_8blocks_ghash_current_128_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline(never)] +/// Encrypt complete 128-byte AES-128-GCM chunks and fold their ciphertext into GHASH. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available; `data` contains at least 128 bytes; and the round keys, +/// counter prefix, accumulator, and GHASH tables belong to the same GCM key and stream. pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_128_core( keys: &Ce128RoundKeys, iv_prefix: &[u8; 12], @@ -2070,7 +2187,7 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_128_core( tables.h_powers_rev_16.as_ptr(), tables.h_powers_rev_16_mid.as_ptr(), tables.h_powers_rev_16_pair.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -2084,7 +2201,7 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_128_core( data.as_mut_ptr(), data.len(), tables.h_powers_rev_8.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -2125,6 +2242,12 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_128_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline(never)] +/// Fold and decrypt complete 128-byte AES-128-GCM ciphertext chunks. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available and the round keys, counter prefix, accumulator, and +/// GHASH tables belong to the same GCM key and stream. `data` may be shorter than one complete chunk. pub(super) unsafe fn decrypt_ctr32_be_xor_ghash_128b_chunks_128_core( keys: &Ce128RoundKeys, iv_prefix: &[u8; 12], @@ -2152,7 +2275,7 @@ pub(super) unsafe fn decrypt_ctr32_be_xor_ghash_128b_chunks_128_core( tables.h_powers_rev_16.as_ptr(), tables.h_powers_rev_16_mid.as_ptr(), tables.h_powers_rev_16_pair.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -2166,7 +2289,7 @@ pub(super) unsafe fn decrypt_ctr32_be_xor_ghash_128b_chunks_128_core( data.as_mut_ptr(), data.len(), tables.h_powers_rev_8.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); diff --git a/src/aead/aes/riscv64_fixslice_aes.rs b/src/aead/aes/riscv64_fixslice_aes.rs index 19002124..5c896fe5 100644 --- a/src/aead/aes/riscv64_fixslice_aes.rs +++ b/src/aead/aes/riscv64_fixslice_aes.rs @@ -8,8 +8,6 @@ //! Reference: Adomnicai et al., "Fixslicing AES-like Ciphers", //! . -#![allow(clippy::unreadable_literal)] - use super::{BLOCK_SIZE, KEY_SIZE, KEY_SIZE_128}; type State = [u64; 8]; @@ -27,7 +25,7 @@ impl RvFixsliceRoundKeys { } #[inline] - #[allow(dead_code)] + #[cfg(target_arch = "riscv64")] pub(super) fn zeroize(&mut self) { // SAFETY: `[u64; 120]` is contiguous and valid to view as bytes for its // exact initialized size. @@ -51,7 +49,7 @@ impl RvFixslice128RoundKeys { } #[inline] - #[allow(dead_code)] + #[cfg(target_arch = "riscv64")] pub(super) fn zeroize(&mut self) { // SAFETY: `[u64; 88]` is contiguous and valid to view as bytes for its // exact initialized size. @@ -110,7 +108,6 @@ pub(super) fn encrypt_4blocks(rkeys: &RvFixsliceRoundKeys, blocks: &mut [[u8; BL } #[inline] -#[allow(dead_code)] pub(super) fn cipher_round_4(blocks: &mut [[u8; BLOCK_SIZE]; 4], round_keys: &[[u8; BLOCK_SIZE]; 4]) { let mut state = State::default(); bitslice(&mut state, &blocks[0], &blocks[1], &blocks[2], &blocks[3]); @@ -625,14 +622,15 @@ fn inv_bitslice(input: &[u64]) -> [[u8; BLOCK_SIZE]; 4] { #[rustfmt::skip] fn write_reordered(columns: u64, output: &mut [u8]) { - output[0x0] = (columns ) as u8; - output[0x1] = (columns >> 0x10) as u8; - output[0x2] = (columns >> 0x20) as u8; - output[0x3] = (columns >> 0x30) as u8; - output[0x8] = (columns >> 0x08) as u8; - output[0x9] = (columns >> 0x18) as u8; - output[0xa] = (columns >> 0x28) as u8; - output[0xb] = (columns >> 0x38) as u8; + let bytes = columns.to_le_bytes(); + output[0x0] = bytes[0]; + output[0x1] = bytes[2]; + output[0x2] = bytes[4]; + output[0x3] = bytes[6]; + output[0x8] = bytes[1]; + output[0x9] = bytes[3]; + output[0xa] = bytes[5]; + output[0xb] = bytes[7]; } let mut output = [[0u8; BLOCK_SIZE]; 4]; @@ -691,7 +689,7 @@ fn ror(x: u64, y: u32) -> u64 { #[inline(always)] fn ror_distance(rows: u32, cols: u32) -> u32 { - (rows << 4) + (cols << 2) + rows.strict_shl(4).strict_add(cols.strict_shl(2)) } #[inline(always)] diff --git a/src/aead/aes/riscv64_vperm_aes.rs b/src/aead/aes/riscv64_vperm_aes.rs deleted file mode 100644 index f48333f8..00000000 --- a/src/aead/aes/riscv64_vperm_aes.rs +++ /dev/null @@ -1,617 +0,0 @@ -use core::arch::asm; - -use crate::aead::aes_round::{ - AES_AFFINE, MC_ROT1, MC_ROT2, VPERM_INV_HI, VPERM_INV_LO, VPERM_IPT_HI, VPERM_IPT_LO, VPERM_SBOT, VPERM_SBOU, - VPERM_SR, XTIME_REDUCE, -}; - -/// Precomputed Hamburg vperm table block — contiguous for offset-based loads. -#[repr(C, align(16))] -struct VpermTables { - ipt_lo: [u8; 16], // offset 0 - ipt_hi: [u8; 16], // offset 16 - inv_lo: [u8; 16], // offset 32 - inv_hi: [u8; 16], // offset 48 - sbou: [u8; 16], // offset 64 - sbot: [u8; 16], // offset 80 - sr_perm: [u8; 16], // offset 96 - mc_rot1: [u8; 16], // offset 112 - mc_rot2: [u8; 16], // offset 128 - affine: [u8; 16], // offset 144 - xtime: [u8; 16], // offset 160 -} - -impl VpermTables { - #[inline(always)] - fn load() -> Self { - Self { - ipt_lo: VPERM_IPT_LO, - ipt_hi: VPERM_IPT_HI, - inv_lo: VPERM_INV_LO, - inv_hi: VPERM_INV_HI, - sbou: VPERM_SBOU, - sbot: VPERM_SBOT, - sr_perm: VPERM_SR, - mc_rot1: MC_ROT1, - mc_rot2: MC_ROT2, - affine: [AES_AFFINE; 16], - xtime: [XTIME_REDUCE; 16], - } - } -} - -/// Extract round key bytes from the portable key schedule. -#[inline] -fn round_key_bytes(rk: &[u32; super::EXPANDED_KEY_WORDS], round: usize) -> [u8; 16] { - let off = round.strict_mul(4); - let mut bytes = [0u8; 16]; - bytes[0..4].copy_from_slice(&rk[off].to_be_bytes()); - bytes[4..8].copy_from_slice(&rk[off.strict_add(1)].to_be_bytes()); - bytes[8..12].copy_from_slice(&rk[off.strict_add(2)].to_be_bytes()); - bytes[12..16].copy_from_slice(&rk[off.strict_add(3)].to_be_bytes()); - bytes -} - -/// Extract round key bytes from the AES-128 portable key schedule. -#[inline] -fn round_key_bytes_128(rk: &[u32; super::EXPANDED_KEY_WORDS_128], round: usize) -> [u8; 16] { - let off = round.strict_mul(4); - let mut bytes = [0u8; 16]; - bytes[0..4].copy_from_slice(&rk[off].to_be_bytes()); - bytes[4..8].copy_from_slice(&rk[off.strict_add(1)].to_be_bytes()); - bytes[8..12].copy_from_slice(&rk[off.strict_add(2)].to_be_bytes()); - bytes[12..16].copy_from_slice(&rk[off.strict_add(3)].to_be_bytes()); - bytes -} - -macro_rules! vperm_inner_round_m1 { - ($state:literal, $rk:literal) => { - concat!( - "vand.vi v19, ", - $state, - ", 15\n", - "vsrl.vi v20, ", - $state, - ", 4\n", - "vrgather.vv v21, v8, v19\n", - "vrgather.vv v22, v9, v20\n", - "vxor.vv v19, v21, v22\n", - "vand.vi v20, v19, 15\n", - "vsrl.vi v21, v19, 4\n", - "vrgather.vv v22, v11, v20\n", - "vxor.vv v23, v21, v20\n", - "vrgather.vv v24, v10, v21\n", - "vxor.vv v25, v24, v22\n", - "vrgather.vv v24, v10, v23\n", - "vxor.vv v26, v24, v22\n", - "vand.vi v27, v25, 15\n", - "vrgather.vv v28, v10, v27\n", - "vsra.vi v29, v25, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v28, v28, v29\n", - "vxor.vv v19, v28, v23\n", - "vand.vi v27, v26, 15\n", - "vrgather.vv v28, v10, v27\n", - "vsra.vi v29, v26, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v28, v28, v29\n", - "vxor.vv v20, v28, v21\n", - "vand.vi v27, v19, 15\n", - "vrgather.vv v28, v12, v27\n", - "vsra.vi v29, v19, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v22, v28, v29\n", - "vand.vi v27, v20, 15\n", - "vrgather.vv v30, v13, v27\n", - "vsra.vi v29, v20, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v23, v30, v29\n", - "vxor.vv v19, v22, v23\n", - "vxor.vv v19, v19, v17\n", - "vrgather.vv v20, v19, v14\n", - "vrgather.vv v21, v20, v15\n", - "vxor.vv v22, v20, v21\n", - "vsll.vi v23, v22, 1\n", - "vsra.vi v24, v22, 7\n", - "vand.vv v24, v24, v18\n", - "vxor.vv v23, v23, v24\n", - "vrgather.vv v24, v22, v16\n", - "vxor.vv v25, v22, v24\n", - "vxor.vv v26, v20, v25\n", - "vxor.vv v26, v26, v23\n", - "vxor.vv ", - $state, - ", v26, ", - $rk - ) - }; -} - -macro_rules! vperm_final_round_m1 { - ($state:literal, $rk:literal) => { - concat!( - "vand.vi v19, ", - $state, - ", 15\n", - "vsrl.vi v20, ", - $state, - ", 4\n", - "vrgather.vv v21, v8, v19\n", - "vrgather.vv v22, v9, v20\n", - "vxor.vv v19, v21, v22\n", - "vand.vi v20, v19, 15\n", - "vsrl.vi v21, v19, 4\n", - "vrgather.vv v22, v11, v20\n", - "vxor.vv v23, v21, v20\n", - "vrgather.vv v24, v10, v21\n", - "vxor.vv v25, v24, v22\n", - "vrgather.vv v24, v10, v23\n", - "vxor.vv v26, v24, v22\n", - "vand.vi v27, v25, 15\n", - "vrgather.vv v28, v10, v27\n", - "vsra.vi v29, v25, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v28, v28, v29\n", - "vxor.vv v19, v28, v23\n", - "vand.vi v27, v26, 15\n", - "vrgather.vv v28, v10, v27\n", - "vsra.vi v29, v26, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v28, v28, v29\n", - "vxor.vv v20, v28, v21\n", - "vand.vi v27, v19, 15\n", - "vrgather.vv v28, v12, v27\n", - "vsra.vi v29, v19, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v22, v28, v29\n", - "vand.vi v27, v20, 15\n", - "vrgather.vv v30, v13, v27\n", - "vsra.vi v29, v20, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v23, v30, v29\n", - "vxor.vv v19, v22, v23\n", - "vxor.vv v19, v19, v17\n", - "vrgather.vv v20, v19, v14\n", - "vxor.vv ", - $state, - ", v20, ", - $rk - ) - }; -} - -/// Single inner AES round (SubBytes + ShiftRows + MixColumns + AddRoundKey). -/// Same asm as aegis256::rv_vperm::aes_round. -#[target_feature(enable = "v")] -#[inline] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -unsafe fn aes_inner_round(block: &[u8; 16], round_key: &[u8; 16], tables: &VpermTables) -> [u8; 16] { - let mut out = [0u8; 16]; - // SAFETY: caller guarantees the RISC-V V extension is available and the - // asm block only reads the provided state/table/key buffers and writes `out`. - unsafe { - asm!( - "vsetivli zero, 16, e8, m1, ta, ma", - // Load tables. - "vle8.v v2, ({tbl})", - "addi {tmp}, {tbl}, 16", - "vle8.v v3, ({tmp})", - "addi {tmp}, {tbl}, 32", - "vle8.v v4, ({tmp})", - "addi {tmp}, {tbl}, 48", - "vle8.v v5, ({tmp})", - "addi {tmp}, {tbl}, 64", - "vle8.v v6, ({tmp})", - "addi {tmp}, {tbl}, 80", - "vle8.v v7, ({tmp})", - "addi {tmp}, {tbl}, 96", - "vle8.v v8, ({tmp})", - "addi {tmp}, {tbl}, 112", - "vle8.v v9, ({tmp})", - "addi {tmp}, {tbl}, 128", - "vle8.v v10, ({tmp})", - "addi {tmp}, {tbl}, 144", - "vle8.v v11, ({tmp})", - "addi {tmp}, {tbl}, 160", - "vle8.v v12, ({tmp})", - "vle8.v v0, ({state})", - "vle8.v v1, ({rk})", - // Phase 1: Nibble extraction. - "vand.vi v14, v0, 15", - "vsrl.vi v15, v0, 4", - // Phase 2: Input transform. - "vrgather.vv v16, v2, v14", - "vrgather.vv v17, v3, v15", - "vxor.vv v14, v16, v17", - // Phase 3: Re-extract nibbles. - "vand.vi v15, v14, 15", - "vsrl.vi v16, v14, 4", - // Phase 4: GF(2^4) inverse. - "vrgather.vv v17, v5, v15", - "vxor.vv v18, v16, v15", - "vrgather.vv v19, v4, v16", - "vxor.vv v20, v19, v17", - "vrgather.vv v21, v4, v18", - "vxor.vv v22, v21, v17", - "vand.vi v23, v20, 15", - "vrgather.vv v24, v4, v23", - "vsra.vi v25, v20, 7", - "vxor.vi v26, v25, -1", - "vand.vv v24, v24, v26", - "vxor.vv v14, v24, v18", - "vand.vi v23, v22, 15", - "vrgather.vv v24, v4, v23", - "vsra.vi v25, v22, 7", - "vxor.vi v26, v25, -1", - "vand.vv v24, v24, v26", - "vxor.vv v15, v24, v16", - // Phase 5: Output transform. - "vand.vi v23, v14, 15", - "vrgather.vv v24, v6, v23", - "vsra.vi v25, v14, 7", - "vxor.vi v26, v25, -1", - "vand.vv v16, v24, v26", - "vand.vi v23, v15, 15", - "vrgather.vv v24, v7, v23", - "vsra.vi v25, v15, 7", - "vxor.vi v26, v25, -1", - "vand.vv v17, v24, v26", - "vxor.vv v14, v16, v17", - "vxor.vv v14, v14, v11", - // ShiftRows. - "vrgather.vv v15, v14, v8", - // MixColumns. - "vrgather.vv v16, v15, v9", - "vxor.vv v17, v15, v16", - "vsll.vi v18, v17, 1", - "vsra.vi v19, v17, 7", - "vand.vv v19, v19, v12", - "vxor.vv v18, v18, v19", - "vrgather.vv v19, v17, v10", - "vxor.vv v20, v17, v19", - "vxor.vv v14, v15, v20", - "vxor.vv v14, v14, v18", - // AddRoundKey. - "vxor.vv v0, v14, v1", - "vse8.v v0, ({out})", - state = in(reg) block.as_ptr(), - rk = in(reg) round_key.as_ptr(), - tbl = in(reg) tables as *const VpermTables as *const u8, - out = in(reg) out.as_mut_ptr(), - tmp = out(reg) _, - options(nostack), - ); - } - out -} - -/// Final AES round (SubBytes + ShiftRows + AddRoundKey, no MixColumns). -#[target_feature(enable = "v")] -#[inline] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -unsafe fn aes_final_round(block: &[u8; 16], round_key: &[u8; 16], tables: &VpermTables) -> [u8; 16] { - let mut out = [0u8; 16]; - // SAFETY: caller guarantees the RISC-V V extension is available and the - // asm block only reads the provided state/table/key buffers and writes `out`. - unsafe { - asm!( - "vsetivli zero, 16, e8, m1, ta, ma", - // Load tables (only need SubBytes + ShiftRows, no MixColumns). - "vle8.v v2, ({tbl})", - "addi {tmp}, {tbl}, 16", - "vle8.v v3, ({tmp})", - "addi {tmp}, {tbl}, 32", - "vle8.v v4, ({tmp})", - "addi {tmp}, {tbl}, 48", - "vle8.v v5, ({tmp})", - "addi {tmp}, {tbl}, 64", - "vle8.v v6, ({tmp})", - "addi {tmp}, {tbl}, 80", - "vle8.v v7, ({tmp})", - "addi {tmp}, {tbl}, 96", - "vle8.v v8, ({tmp})", - "addi {tmp}, {tbl}, 144", - "vle8.v v11, ({tmp})", - "vle8.v v0, ({state})", - "vle8.v v1, ({rk})", - // SubBytes (same as inner round). - "vand.vi v14, v0, 15", - "vsrl.vi v15, v0, 4", - "vrgather.vv v16, v2, v14", - "vrgather.vv v17, v3, v15", - "vxor.vv v14, v16, v17", - "vand.vi v15, v14, 15", - "vsrl.vi v16, v14, 4", - "vrgather.vv v17, v5, v15", - "vxor.vv v18, v16, v15", - "vrgather.vv v19, v4, v16", - "vxor.vv v20, v19, v17", - "vrgather.vv v21, v4, v18", - "vxor.vv v22, v21, v17", - "vand.vi v23, v20, 15", - "vrgather.vv v24, v4, v23", - "vsra.vi v25, v20, 7", - "vxor.vi v26, v25, -1", - "vand.vv v24, v24, v26", - "vxor.vv v14, v24, v18", - "vand.vi v23, v22, 15", - "vrgather.vv v24, v4, v23", - "vsra.vi v25, v22, 7", - "vxor.vi v26, v25, -1", - "vand.vv v24, v24, v26", - "vxor.vv v15, v24, v16", - "vand.vi v23, v14, 15", - "vrgather.vv v24, v6, v23", - "vsra.vi v25, v14, 7", - "vxor.vi v26, v25, -1", - "vand.vv v16, v24, v26", - "vand.vi v23, v15, 15", - "vrgather.vv v24, v7, v23", - "vsra.vi v25, v15, 7", - "vxor.vi v26, v25, -1", - "vand.vv v17, v24, v26", - "vxor.vv v14, v16, v17", - "vxor.vv v14, v14, v11", - // ShiftRows (no MixColumns). - "vrgather.vv v15, v14, v8", - // AddRoundKey. - "vxor.vv v0, v15, v1", - "vse8.v v0, ({out})", - state = in(reg) block.as_ptr(), - rk = in(reg) round_key.as_ptr(), - tbl = in(reg) tables as *const VpermTables as *const u8, - out = in(reg) out.as_mut_ptr(), - tmp = out(reg) _, - options(nostack), - ); - } - out -} - -/// Four independent inner rounds with one shared table load. -/// -/// This targets the AES-GCM-SIV fixed-cost path, where six ECB blocks are -/// derived up front. Sharing the vperm tables across 4 blocks removes most -/// of the per-call table-load overhead even before we attempt a wider LMUL -/// kernel. -#[target_feature(enable = "v")] -#[inline] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -unsafe fn aes_inner_round_4(blocks: &mut [[u8; 16]; 4], round_key: &[u8; 16], tables: &VpermTables) { - // SAFETY: caller guarantees the RISC-V V extension is available and all - // block/key/table references are valid for 16-byte vector loads/stores. - unsafe { - asm!( - "vsetivli zero, 16, e8, m1, ta, ma", - "vle8.v v8, ({tbl})", - "addi {tmp}, {tbl}, 16", - "vle8.v v9, ({tmp})", - "addi {tmp}, {tbl}, 32", - "vle8.v v10, ({tmp})", - "addi {tmp}, {tbl}, 48", - "vle8.v v11, ({tmp})", - "addi {tmp}, {tbl}, 64", - "vle8.v v12, ({tmp})", - "addi {tmp}, {tbl}, 80", - "vle8.v v13, ({tmp})", - "addi {tmp}, {tbl}, 96", - "vle8.v v14, ({tmp})", - "addi {tmp}, {tbl}, 112", - "vle8.v v15, ({tmp})", - "addi {tmp}, {tbl}, 128", - "vle8.v v16, ({tmp})", - "addi {tmp}, {tbl}, 144", - "vle8.v v17, ({tmp})", - "addi {tmp}, {tbl}, 160", - "vle8.v v18, ({tmp})", - "vle8.v v0, ({b0})", - "vle8.v v1, ({b1})", - "vle8.v v2, ({b2})", - "vle8.v v3, ({b3})", - "vle8.v v4, ({rk})", - vperm_inner_round_m1!("v0", "v4"), - vperm_inner_round_m1!("v1", "v4"), - vperm_inner_round_m1!("v2", "v4"), - vperm_inner_round_m1!("v3", "v4"), - "vse8.v v0, ({b0})", - "vse8.v v1, ({b1})", - "vse8.v v2, ({b2})", - "vse8.v v3, ({b3})", - b0 = in(reg) blocks[0].as_mut_ptr(), - b1 = in(reg) blocks[1].as_mut_ptr(), - b2 = in(reg) blocks[2].as_mut_ptr(), - b3 = in(reg) blocks[3].as_mut_ptr(), - rk = in(reg) round_key.as_ptr(), - tbl = in(reg) tables as *const VpermTables as *const u8, - tmp = out(reg) _, - options(nostack), - ); - } -} - -/// Four independent final rounds with one shared table load. -#[target_feature(enable = "v")] -#[inline] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -unsafe fn aes_final_round_4(blocks: &mut [[u8; 16]; 4], round_key: &[u8; 16], tables: &VpermTables) { - // SAFETY: caller guarantees the RISC-V V extension is available and all - // block/key/table references are valid for 16-byte vector loads/stores. - unsafe { - asm!( - "vsetivli zero, 16, e8, m1, ta, ma", - "vle8.v v8, ({tbl})", - "addi {tmp}, {tbl}, 16", - "vle8.v v9, ({tmp})", - "addi {tmp}, {tbl}, 32", - "vle8.v v10, ({tmp})", - "addi {tmp}, {tbl}, 48", - "vle8.v v11, ({tmp})", - "addi {tmp}, {tbl}, 64", - "vle8.v v12, ({tmp})", - "addi {tmp}, {tbl}, 80", - "vle8.v v13, ({tmp})", - "addi {tmp}, {tbl}, 96", - "vle8.v v14, ({tmp})", - "addi {tmp}, {tbl}, 144", - "vle8.v v17, ({tmp})", - "vle8.v v0, ({b0})", - "vle8.v v1, ({b1})", - "vle8.v v2, ({b2})", - "vle8.v v3, ({b3})", - "vle8.v v4, ({rk})", - vperm_final_round_m1!("v0", "v4"), - vperm_final_round_m1!("v1", "v4"), - vperm_final_round_m1!("v2", "v4"), - vperm_final_round_m1!("v3", "v4"), - "vse8.v v0, ({b0})", - "vse8.v v1, ({b1})", - "vse8.v v2, ({b2})", - "vse8.v v3, ({b3})", - b0 = in(reg) blocks[0].as_mut_ptr(), - b1 = in(reg) blocks[1].as_mut_ptr(), - b2 = in(reg) blocks[2].as_mut_ptr(), - b3 = in(reg) blocks[3].as_mut_ptr(), - rk = in(reg) round_key.as_ptr(), - tbl = in(reg) tables as *const VpermTables as *const u8, - tmp = out(reg) _, - options(nostack), - ); - } -} - -/// AES-256 full-block encryption (14 rounds) using Hamburg vperm. -/// -/// # Safety -/// Requires the RISC-V V extension. -#[target_feature(enable = "v")] -pub(super) unsafe fn encrypt_block(rk: &[u32; super::EXPANDED_KEY_WORDS], block: &mut [u8; 16]) { - // SAFETY: caller guarantees the RISC-V V extension is available for the - // full AES-256 block operation and all references are valid Rust buffers. - unsafe { - let tables = VpermTables::load(); - - // Initial AddRoundKey (round 0). - let rk0 = round_key_bytes(rk, 0); - for i in 0..16 { - block[i] ^= rk0[i]; - } - - // Rounds 1-13: full AES round (SubBytes + ShiftRows + MixColumns + AddRoundKey). - let mut round = 1usize; - while round < super::ROUNDS { - let rk_r = round_key_bytes(rk, round); - *block = aes_inner_round(block, &rk_r, &tables); - round = round.strict_add(1); - } - - // Round 14 (final): SubBytes + ShiftRows + AddRoundKey (no MixColumns). - let rk14 = round_key_bytes(rk, super::ROUNDS); - *block = aes_final_round(block, &rk14, &tables); - } -} - -/// Encrypt 4 independent AES-256 blocks using the vperm backend. -/// -/// The 4-block batch is aimed at GCM-SIV key derivation, which produces 6 -/// unrelated ECB inputs up front. Processing 4 of them together amortizes -/// table loads across the hot fixed-cost path while keeping the existing -/// single-block kernel for small tails. -#[target_feature(enable = "v")] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -pub(super) unsafe fn encrypt_4blocks(rk: &[u32; super::EXPANDED_KEY_WORDS], blocks: &mut [[u8; 16]; 4]) { - // SAFETY: caller guarantees the RISC-V V extension is available for the - // duration of the batch and all block buffers are valid 16-byte arrays. - unsafe { - let tables = VpermTables::load(); - - let rk0 = round_key_bytes(rk, 0); - for block in blocks.iter_mut() { - for i in 0..16 { - block[i] ^= rk0[i]; - } - } - - let mut round = 1usize; - while round < super::ROUNDS { - let rk_r = round_key_bytes(rk, round); - aes_inner_round_4(blocks, &rk_r, &tables); - round = round.strict_add(1); - } - - let rk14 = round_key_bytes(rk, super::ROUNDS); - aes_final_round_4(blocks, &rk14, &tables); - } -} - -// AES-128 (10 rounds) - -/// AES-128 full-block encryption (10 rounds) using Hamburg vperm. -/// -/// # Safety -/// Requires the RISC-V V extension. -#[target_feature(enable = "v")] -pub(super) unsafe fn encrypt_block_128(rk: &[u32; super::EXPANDED_KEY_WORDS_128], block: &mut [u8; 16]) { - // SAFETY: caller guarantees the RISC-V V extension is available for the - // full AES-128 block operation and all references are valid Rust buffers. - unsafe { - let tables = VpermTables::load(); - - // Initial AddRoundKey (round 0). - let rk0 = round_key_bytes_128(rk, 0); - for i in 0..16 { - block[i] ^= rk0[i]; - } - - // Rounds 1-9: full AES round (SubBytes + ShiftRows + MixColumns + AddRoundKey). - let mut round = 1usize; - while round < super::ROUNDS_128 { - let rk_r = round_key_bytes_128(rk, round); - *block = aes_inner_round(block, &rk_r, &tables); - round = round.strict_add(1); - } - - // Round 10 (final): SubBytes + ShiftRows + AddRoundKey (no MixColumns). - let rk10 = round_key_bytes_128(rk, super::ROUNDS_128); - *block = aes_final_round(block, &rk10, &tables); - } -} - -/// Encrypt 4 independent AES-128 blocks using the vperm backend. -/// -/// # Safety -/// Caller must ensure the CPU supports the RISC-V `v` extension. -#[target_feature(enable = "v")] -pub(super) unsafe fn encrypt_4blocks_128(rk: &[u32; super::EXPANDED_KEY_WORDS_128], blocks: &mut [[u8; 16]; 4]) { - // SAFETY: caller guarantees the RISC-V V extension is available for the - // duration of the batch and all block buffers are valid 16-byte arrays. - unsafe { - let tables = VpermTables::load(); - - let rk0 = round_key_bytes_128(rk, 0); - for block in blocks.iter_mut() { - for i in 0..16 { - block[i] ^= rk0[i]; - } - } - - let mut round = 1usize; - while round < super::ROUNDS_128 { - let rk_r = round_key_bytes_128(rk, round); - aes_inner_round_4(blocks, &rk_r, &tables); - round = round.strict_add(1); - } - - let rk10 = round_key_bytes_128(rk, super::ROUNDS_128); - aes_final_round_4(blocks, &rk10, &tables); - } -} diff --git a/src/aead/aes/x86_64/asm.rs b/src/aead/aes/x86_64/asm.rs index 9a7caef4..c266ed69 100644 --- a/src/aead/aes/x86_64/asm.rs +++ b/src/aead/aes/x86_64/asm.rs @@ -1,13 +1,10 @@ //! Linux x86-64 rscrypto-owned AES-GCM assembly kernels. -#![allow(unsafe_code)] - use core::arch::global_asm; global_asm!(include_str!("asm/rscrypto_aes_gcm_x86_64_linux.s")); #[repr(C)] -#[allow(dead_code)] pub(super) struct AesGcmX86State { acc_lo: u64, acc_hi: u64, @@ -16,13 +13,12 @@ pub(super) struct AesGcmX86State { pub(super) processed: usize, } -#[allow(dead_code)] impl AesGcmX86State { #[inline] pub(super) fn new(acc: u128, ctr: u32) -> Self { Self { - acc_lo: acc as u64, - acc_hi: (acc >> 64) as u64, + acc_lo: u64::try_from(acc & u128::from(u64::MAX)).expect("masked accumulator half fits u64"), + acc_hi: u64::try_from(acc >> 64).expect("shifted accumulator half fits u64"), ctr, _pad: 0, processed: 0, @@ -36,7 +32,6 @@ impl AesGcmX86State { } unsafe extern "C" { - #[allow(dead_code)] pub(super) fn rscrypto_aes128_gcm_seal_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -46,7 +41,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes128_gcm_open_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -56,7 +50,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes128_gcm_seal_64x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -66,7 +59,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes128_gcm_open_64x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -76,7 +68,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes128_gcm_seal_128x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -86,7 +77,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes128_gcm_open_128x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -96,7 +86,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes256_gcm_seal_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -106,7 +95,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes256_gcm_open_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -116,7 +104,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes256_gcm_seal_64x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -126,7 +113,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes256_gcm_open_64x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -136,7 +122,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes256_gcm_seal_128x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -146,7 +131,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes256_gcm_open_128x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -156,47 +140,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] - pub(super) fn rscrypto_aes128_gcm_seal_8x_vaes256_x86_64_linux( - round_keys: *const u8, - initial_counter: *const u8, - data: *mut u8, - len: usize, - h_powers_rev_8: *const u128, - state: *mut AesGcmX86State, - ); - - #[allow(dead_code)] - pub(super) fn rscrypto_aes128_gcm_open_8x_vaes256_x86_64_linux( - round_keys: *const u8, - initial_counter: *const u8, - data: *mut u8, - len: usize, - h_powers_rev_8: *const u128, - state: *mut AesGcmX86State, - ); - - #[allow(dead_code)] - pub(super) fn rscrypto_aes256_gcm_seal_8x_vaes256_x86_64_linux( - round_keys: *const u8, - initial_counter: *const u8, - data: *mut u8, - len: usize, - h_powers_rev_8: *const u128, - state: *mut AesGcmX86State, - ); - - #[allow(dead_code)] - pub(super) fn rscrypto_aes256_gcm_open_8x_vaes256_x86_64_linux( - round_keys: *const u8, - initial_counter: *const u8, - data: *mut u8, - len: usize, - h_powers_rev_8: *const u128, - state: *mut AesGcmX86State, - ); - - #[allow(dead_code)] pub(super) fn rscrypto_aes128_gcmsiv_ctr_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -204,7 +147,6 @@ unsafe extern "C" { len: usize, ) -> usize; - #[allow(dead_code)] pub(super) fn rscrypto_aes256_gcmsiv_ctr_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, diff --git a/src/aead/aes/x86_64/asm/rscrypto_aes_gcm_x86_64_linux.s b/src/aead/aes/x86_64/asm/rscrypto_aes_gcm_x86_64_linux.s index a5fe46de..f5a7d1a7 100644 --- a/src/aead/aes/x86_64/asm/rscrypto_aes_gcm_x86_64_linux.s +++ b/src/aead/aes/x86_64/asm/rscrypto_aes_gcm_x86_64_linux.s @@ -79,53 +79,6 @@ vmovdqu32 zmm3{{k1}}, zmm25 .endm -.macro GCM_COUNTERS_8_SLOW - vpbroadcastd ymm25, r10d - - vbroadcasti32x4 ymm0, XMMWORD PTR [rsi] - vmovdqa64 ymm1, ymm0 - vmovdqa64 ymm2, ymm0 - vmovdqa64 ymm3, ymm0 - - vmovdqa32 ymm24, ymm25 - vpaddd ymm24, ymm24, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_inc_y0] - vpshufb ymm24, ymm24, ymm26 - vmovdqu32 ymm0{{k1}}, ymm24 - - vmovdqa32 ymm24, ymm25 - vpaddd ymm24, ymm24, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_inc_y1] - vpshufb ymm24, ymm24, ymm26 - vmovdqu32 ymm1{{k1}}, ymm24 - - vmovdqa32 ymm24, ymm25 - vpaddd ymm24, ymm24, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_inc_y2] - vpshufb ymm24, ymm24, ymm26 - vmovdqu32 ymm2{{k1}}, ymm24 - - vpaddd ymm25, ymm25, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_inc_y3] - vpshufb ymm25, ymm25, ymm26 - vmovdqu32 ymm3{{k1}}, ymm25 -.endm - -.macro GCM_COUNTERS_8 - mov eax, r10d - cmp al, 249 - jae 1f - bswap eax - vpbroadcastd ymm24, eax - - vbroadcasti32x4 ymm25, XMMWORD PTR [rsi] - vmovdqu32 ymm25{{k1}}, ymm24 - - vpaddd ymm0, ymm25, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_incbe_y0] - vpaddd ymm1, ymm25, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_incbe_y1] - vpaddd ymm2, ymm25, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_incbe_y2] - vpaddd ymm3, ymm25, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_incbe_y3] - jmp 2f -1: - GCM_COUNTERS_8_SLOW -2: -.endm .macro AES_ROUND off vbroadcasti32x4 zmm27, XMMWORD PTR [rdi + \off] @@ -173,51 +126,6 @@ .endif .endm -.macro AES_Y_ROUND off - vbroadcasti32x4 ymm27, XMMWORD PTR [rdi + \off] - vaesenc ymm0, ymm0, ymm27 - vaesenc ymm1, ymm1, ymm27 - vaesenc ymm2, ymm2, ymm27 - vaesenc ymm3, ymm3, ymm27 -.endm - -.macro AES_Y_START - vbroadcasti32x4 ymm27, XMMWORD PTR [rdi] - vpxord ymm0, ymm0, ymm27 - vpxord ymm1, ymm1, ymm27 - vpxord ymm2, ymm2, ymm27 - vpxord ymm3, ymm3, ymm27 -.endm - -.macro AES_Y_LAST off - vbroadcasti32x4 ymm27, XMMWORD PTR [rdi + \off] - vaesenclast ymm0, ymm0, ymm27 - vaesenclast ymm1, ymm1, ymm27 - vaesenclast ymm2, ymm2, ymm27 - vaesenclast ymm3, ymm3, ymm27 -.endm - -.macro AES_ENCRYPT_8Y aes256 - AES_Y_START - AES_Y_ROUND 16 - AES_Y_ROUND 32 - AES_Y_ROUND 48 - AES_Y_ROUND 64 - AES_Y_ROUND 80 - AES_Y_ROUND 96 - AES_Y_ROUND 112 - AES_Y_ROUND 128 - AES_Y_ROUND 144 - .if \aes256 - AES_Y_ROUND 160 - AES_Y_ROUND 176 - AES_Y_ROUND 192 - AES_Y_ROUND 208 - AES_Y_LAST 224 - .else - AES_Y_LAST 160 - .endif -.endm .macro GHASH_FOLD data, h vpclmulqdq zmm21, \data, \h, 0x00 @@ -263,22 +171,6 @@ vpxorq zmm29, zmm29, zmm22 .endm -.macro GHASH_FOLD_Y data, h - vpclmulqdq ymm21, \data, \h, 0x00 - vpclmulqdq ymm22, \data, \h, 0x11 - - vpshufd ymm23, \data, 0x4e - vpxord ymm23, ymm23, \data - vpshufd ymm24, \h, 0x4e - vpxord ymm24, ymm24, \h - vpclmulqdq ymm23, ymm23, ymm24, 0x00 - vpternlogq ymm23, ymm21, ymm22, 0x96 - - vpslldq ymm24, ymm23, 8 - vpternlogq ymm28, ymm21, ymm24, 0x96 - vpsrldq ymm24, ymm23, 8 - vpternlogq ymm29, ymm22, ymm24, 0x96 -.endm .macro GHASH_PREP d0, d1, d2, d3 vpshufb \d0, \d0, zmm31 @@ -293,19 +185,6 @@ vpxord zmm29, zmm29, zmm29 .endm -.macro GHASH_PREP_Y d0, d1, d2, d3 - vpshufb \d0, \d0, ymm31 - vpshufb \d1, \d1, ymm31 - vpshufb \d2, \d2, ymm31 - vpshufb \d3, \d3, ymm31 - - vpxord ymm30, ymm30, ymm30 - vmovdqu64 xmm30, XMMWORD PTR [r9] - vpxord \d0, \d0, ymm30 - - vpxord ymm28, ymm28, ymm28 - vpxord ymm29, ymm29, ymm29 -.endm .macro GHASH_REDUCE_ASSEMBLE vpxorq zmm20, zmm20, zmm28 @@ -369,37 +248,6 @@ GHASH_REDUCE_STORE .endm -.macro GHASH_REDUCE_Y - vmovdqa64 xmm16, xmm28 - vextracti64x2 xmm17, ymm28, 1 - vpxorq xmm16, xmm16, xmm17 - - vmovdqa64 xmm19, xmm29 - vextracti64x2 xmm17, ymm29, 1 - vpxorq xmm19, xmm19, xmm17 - - vpsllq xmm17, xmm16, 63 - vpsllq xmm18, xmm16, 62 - vpsllq xmm20, xmm16, 57 - vpternlogq xmm17, xmm18, xmm20, 0x96 - vpslldq xmm17, xmm17, 8 - vpxorq xmm16, xmm16, xmm17 - - vpsrlq xmm17, xmm16, 1 - vpsrlq xmm18, xmm16, 2 - vpsrlq xmm20, xmm16, 7 - vpternlogq xmm17, xmm18, xmm20, 0x96 - vpxorq xmm17, xmm17, xmm16 - - vpsllq xmm18, xmm16, 63 - vpsllq xmm20, xmm16, 62 - vpsllq xmm21, xmm16, 57 - vpternlogq xmm18, xmm20, xmm21, 0x96 - vpsrldq xmm18, xmm18, 8 - - vpternlogq xmm17, xmm18, xmm19, 0x96 - vmovdqu64 XMMWORD PTR [r9], xmm17 -.endm .macro GHASH16_REG d0, d1, d2, d3 GHASH16_REG_H \d0, \d1, \d2, \d3, zmm12, zmm13, zmm14, zmm15 @@ -607,14 +455,6 @@ vmovdqu64 zmm19, ZMMWORD PTR [r8 + 1984] .endm -.macro GHASH8Y_REG d0, d1, d2, d3 - GHASH_PREP_Y \d0, \d1, \d2, \d3 - GHASH_FOLD_Y \d0, ymm12 - GHASH_FOLD_Y \d1, ymm13 - GHASH_FOLD_Y \d2, ymm14 - GHASH_FOLD_Y \d3, ymm15 - GHASH_REDUCE_Y -.endm .macro AES_ENCRYPT_16_WITH_GHASH aes256, d0, d1, d2, d3 AES_START @@ -896,34 +736,6 @@ .endif .endm -.macro AES_ENCRYPT_8Y_WITH_GHASH aes256, d0, d1, d2, d3 - AES_Y_START - GHASH_PREP_Y \d0, \d1, \d2, \d3 - AES_Y_ROUND 16 - GHASH_FOLD_Y \d0, ymm12 - AES_Y_ROUND 32 - GHASH_FOLD_Y \d1, ymm13 - AES_Y_ROUND 48 - GHASH_FOLD_Y \d2, ymm14 - AES_Y_ROUND 64 - GHASH_FOLD_Y \d3, ymm15 - AES_Y_ROUND 80 - AES_Y_ROUND 96 - AES_Y_ROUND 112 - AES_Y_ROUND 128 - AES_Y_ROUND 144 - .if \aes256 - AES_Y_ROUND 160 - AES_Y_ROUND 176 - AES_Y_ROUND 192 - AES_Y_ROUND 208 - GHASH_REDUCE_Y - AES_Y_LAST 224 - .else - GHASH_REDUCE_Y - AES_Y_LAST 160 - .endif -.endm .macro AES_GCM_STORE_Z open, d0, d1, d2, d3 .if \open @@ -1573,114 +1385,6 @@ .size \name, . - \name .endm -.macro AES_GCM_8Y_FUNC name, aes256, open - .p2align 5 - .globl \name - .type \name, @function -\name: - mov r10d, DWORD PTR [r9 + 16] - - cmp rcx, 128 - jb .L\name\()_done - - vmovdqu64 ymm31, YMMWORD PTR [rip + .Lrscrypto_x86_gcm_bswap] - vmovdqu64 ymm26, YMMWORD PTR [rip + .Lrscrypto_x86_dword_bswap] - vmovdqu64 ymm12, YMMWORD PTR [r8] - vmovdqu64 ymm13, YMMWORD PTR [r8 + 32] - vmovdqu64 ymm14, YMMWORD PTR [r8 + 64] - vmovdqu64 ymm15, YMMWORD PTR [r8 + 96] - mov eax, 0x88 - kmovw k1, eax - - GCM_COUNTERS_8 - - AES_ENCRYPT_8Y \aes256 - - .if \open - vmovdqu64 ymm4, YMMWORD PTR [rdx] - vmovdqu64 ymm5, YMMWORD PTR [rdx + 32] - vmovdqu64 ymm6, YMMWORD PTR [rdx + 64] - vmovdqu64 ymm7, YMMWORD PTR [rdx + 96] - vpxord ymm8, ymm0, ymm4 - vpxord ymm9, ymm1, ymm5 - vpxord ymm10, ymm2, ymm6 - vpxord ymm11, ymm3, ymm7 - vmovdqu64 YMMWORD PTR [rdx], ymm8 - vmovdqu64 YMMWORD PTR [rdx + 32], ymm9 - vmovdqu64 YMMWORD PTR [rdx + 64], ymm10 - vmovdqu64 YMMWORD PTR [rdx + 96], ymm11 - .else - vmovdqu64 ymm4, YMMWORD PTR [rdx] - vmovdqu64 ymm5, YMMWORD PTR [rdx + 32] - vmovdqu64 ymm6, YMMWORD PTR [rdx + 64] - vmovdqu64 ymm7, YMMWORD PTR [rdx + 96] - vpxord ymm4, ymm4, ymm0 - vpxord ymm5, ymm5, ymm1 - vpxord ymm6, ymm6, ymm2 - vpxord ymm7, ymm7, ymm3 - vmovdqu64 YMMWORD PTR [rdx], ymm4 - vmovdqu64 YMMWORD PTR [rdx + 32], ymm5 - vmovdqu64 YMMWORD PTR [rdx + 64], ymm6 - vmovdqu64 YMMWORD PTR [rdx + 96], ymm7 - .endif - - add rdx, 128 - sub rcx, 128 - add r10d, 8 - add QWORD PTR [r9 + 24], 128 - cmp rcx, 128 - jb .L\name\()_final_ghash - - .p2align 5 -.L\name\()_loop: - GCM_COUNTERS_8 - - AES_ENCRYPT_8Y_WITH_GHASH \aes256, ymm4, ymm5, ymm6, ymm7 - - .if \open - vmovdqu64 ymm4, YMMWORD PTR [rdx] - vmovdqu64 ymm5, YMMWORD PTR [rdx + 32] - vmovdqu64 ymm6, YMMWORD PTR [rdx + 64] - vmovdqu64 ymm7, YMMWORD PTR [rdx + 96] - vpxord ymm8, ymm0, ymm4 - vpxord ymm9, ymm1, ymm5 - vpxord ymm10, ymm2, ymm6 - vpxord ymm11, ymm3, ymm7 - vmovdqu64 YMMWORD PTR [rdx], ymm8 - vmovdqu64 YMMWORD PTR [rdx + 32], ymm9 - vmovdqu64 YMMWORD PTR [rdx + 64], ymm10 - vmovdqu64 YMMWORD PTR [rdx + 96], ymm11 - .else - vmovdqu64 ymm4, YMMWORD PTR [rdx] - vmovdqu64 ymm5, YMMWORD PTR [rdx + 32] - vmovdqu64 ymm6, YMMWORD PTR [rdx + 64] - vmovdqu64 ymm7, YMMWORD PTR [rdx + 96] - vpxord ymm4, ymm4, ymm0 - vpxord ymm5, ymm5, ymm1 - vpxord ymm6, ymm6, ymm2 - vpxord ymm7, ymm7, ymm3 - vmovdqu64 YMMWORD PTR [rdx], ymm4 - vmovdqu64 YMMWORD PTR [rdx + 32], ymm5 - vmovdqu64 YMMWORD PTR [rdx + 64], ymm6 - vmovdqu64 YMMWORD PTR [rdx + 96], ymm7 - .endif - - add rdx, 128 - sub rcx, 128 - add r10d, 8 - add QWORD PTR [r9 + 24], 128 - cmp rcx, 128 - jae .L\name\()_loop - -.L\name\()_final_ghash: - GHASH8Y_REG ymm4, ymm5, ymm6, ymm7 - -.L\name\()_done: - mov DWORD PTR [r9 + 16], r10d - vzeroupper - ret - .size \name, . - \name -.endm AES_GCM_16X_FUNC rscrypto_aes128_gcm_seal_16x_vaes512_x86_64_linux, 0, 0 AES_GCM_16X_FUNC rscrypto_aes128_gcm_open_16x_vaes512_x86_64_linux, 0, 1 @@ -1696,10 +1400,6 @@ AES_GCM_128X_FUNC rscrypto_aes256_gcm_seal_128x_vaes512_x86_64_linux, 1, 0 AES_GCM_128X_FUNC rscrypto_aes256_gcm_open_128x_vaes512_x86_64_linux, 1, 1 AES_GCMSIV_CTR_16X_FUNC rscrypto_aes128_gcmsiv_ctr_16x_vaes512_x86_64_linux, 0 AES_GCMSIV_CTR_16X_FUNC rscrypto_aes256_gcmsiv_ctr_16x_vaes512_x86_64_linux, 1 -AES_GCM_8Y_FUNC rscrypto_aes128_gcm_seal_8x_vaes256_x86_64_linux, 0, 0 -AES_GCM_8Y_FUNC rscrypto_aes128_gcm_open_8x_vaes256_x86_64_linux, 0, 1 -AES_GCM_8Y_FUNC rscrypto_aes256_gcm_seal_8x_vaes256_x86_64_linux, 1, 0 -AES_GCM_8Y_FUNC rscrypto_aes256_gcm_open_8x_vaes256_x86_64_linux, 1, 1 .section .rodata.cst64,"aM",@progbits,64 .p2align 6 @@ -1745,23 +1445,3 @@ AES_GCM_8Y_FUNC rscrypto_aes256_gcm_open_8x_vaes256_x86_64_linux, 1, 1 .long 8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0 .Lrscrypto_x86_gcmsiv_ctr_inc_z3: .long 12, 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, 0, 0, 0 - -.p2align 5 -.Lrscrypto_x86_ctr_inc_y0: - .long 0, 0, 0, 0, 0, 0, 0, 1 -.Lrscrypto_x86_ctr_inc_y1: - .long 0, 0, 0, 2, 0, 0, 0, 3 -.Lrscrypto_x86_ctr_inc_y2: - .long 0, 0, 0, 4, 0, 0, 0, 5 -.Lrscrypto_x86_ctr_inc_y3: - .long 0, 0, 0, 6, 0, 0, 0, 7 - -.p2align 5 -.Lrscrypto_x86_ctr_incbe_y0: - .long 0, 0, 0, 0x00000000, 0, 0, 0, 0x01000000 -.Lrscrypto_x86_ctr_incbe_y1: - .long 0, 0, 0, 0x02000000, 0, 0, 0, 0x03000000 -.Lrscrypto_x86_ctr_incbe_y2: - .long 0, 0, 0, 0x04000000, 0, 0, 0, 0x05000000 -.Lrscrypto_x86_ctr_incbe_y3: - .long 0, 0, 0, 0x06000000, 0, 0, 0, 0x07000000 diff --git a/src/aead/aes/x86_64_ni.rs b/src/aead/aes/x86_64_ni.rs index f441cc19..d2d697e7 100644 --- a/src/aead/aes/x86_64_ni.rs +++ b/src/aead/aes/x86_64_ni.rs @@ -219,61 +219,6 @@ pub(super) unsafe fn encrypt_16blocks( ) } -/// Encrypt 8 AES-256 blocks as four independent VAES-256 dependency chains. -/// -/// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VAES + AES + SSE2. -#[cfg(feature = "aes-gcm")] -#[target_feature(enable = "aes,sse2,avx2,avx512f,avx512vl,vaes")] -#[inline] -#[allow(dead_code)] -pub(super) unsafe fn encrypt_8blocks_y256( - keys: &NiRoundKeys, - b0: __m256i, - b1: __m256i, - b2: __m256i, - b3: __m256i, -) -> (__m256i, __m256i, __m256i, __m256i) { - let k = &keys.rk; - let rk0 = _mm256_broadcastsi128_si256(k[0]); - let mut s0 = _mm256_xor_si256(b0, rk0); - let mut s1 = _mm256_xor_si256(b1, rk0); - let mut s2 = _mm256_xor_si256(b2, rk0); - let mut s3 = _mm256_xor_si256(b3, rk0); - - macro_rules! round { - ($idx:expr) => {{ - let rk = _mm256_broadcastsi128_si256(k[$idx]); - s0 = _mm256_aesenc_epi128(s0, rk); - s1 = _mm256_aesenc_epi128(s1, rk); - s2 = _mm256_aesenc_epi128(s2, rk); - s3 = _mm256_aesenc_epi128(s3, rk); - }}; - } - - round!(1); - round!(2); - round!(3); - round!(4); - round!(5); - round!(6); - round!(7); - round!(8); - round!(9); - round!(10); - round!(11); - round!(12); - round!(13); - - let rk14 = _mm256_broadcastsi128_si256(k[14]); - ( - _mm256_aesenclast_epi128(s0, rk14), - _mm256_aesenclast_epi128(s1, rk14), - _mm256_aesenclast_epi128(s2, rk14), - _mm256_aesenclast_epi128(s3, rk14), - ) -} - /// Encrypt a single 16-byte block using AES-256 with AES-NI. /// /// # Safety @@ -496,57 +441,6 @@ pub(super) unsafe fn encrypt_16blocks_128( ) } -/// Encrypt 8 AES-128 blocks as four independent VAES-256 dependency chains. -/// -/// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VAES + AES + SSE2. -#[cfg(feature = "aes-gcm")] -#[target_feature(enable = "aes,sse2,avx2,avx512f,avx512vl,vaes")] -#[inline] -#[allow(dead_code)] -pub(super) unsafe fn encrypt_8blocks_128_y256( - keys: &Ni128RoundKeys, - b0: __m256i, - b1: __m256i, - b2: __m256i, - b3: __m256i, -) -> (__m256i, __m256i, __m256i, __m256i) { - let k = &keys.rk; - let rk0 = _mm256_broadcastsi128_si256(k[0]); - let mut s0 = _mm256_xor_si256(b0, rk0); - let mut s1 = _mm256_xor_si256(b1, rk0); - let mut s2 = _mm256_xor_si256(b2, rk0); - let mut s3 = _mm256_xor_si256(b3, rk0); - - macro_rules! round { - ($idx:expr) => {{ - let rk = _mm256_broadcastsi128_si256(k[$idx]); - s0 = _mm256_aesenc_epi128(s0, rk); - s1 = _mm256_aesenc_epi128(s1, rk); - s2 = _mm256_aesenc_epi128(s2, rk); - s3 = _mm256_aesenc_epi128(s3, rk); - }}; - } - - round!(1); - round!(2); - round!(3); - round!(4); - round!(5); - round!(6); - round!(7); - round!(8); - round!(9); - - let rk10 = _mm256_broadcastsi128_si256(k[10]); - ( - _mm256_aesenclast_epi128(s0, rk10), - _mm256_aesenclast_epi128(s1, rk10), - _mm256_aesenclast_epi128(s2, rk10), - _mm256_aesenclast_epi128(s3, rk10), - ) -} - /// Encrypt a single 16-byte block using AES-128 with AES-NI. /// /// # Safety diff --git a/src/aead/aes128gcm.rs b/src/aead/aes128gcm.rs index f3e860fb..a73cc202 100644 --- a/src/aead/aes128gcm.rs +++ b/src/aead/aes128gcm.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! AES-128-GCM public AEAD surface (NIST SP 800-38D). use core::fmt; @@ -277,9 +275,9 @@ fn ghash_update_padded(mut acc: u128, h_polyval: u128, data: &[u8]) -> u128 { #[inline] fn ghash_update_padded_wide(mut acc: u128, h_polyval: u128, h_powers_rev: &[u128; 4], data: &[u8]) -> u128 { let (full_blocks, remainder) = data.as_chunks::<16>(); - let mut chunks = full_blocks.chunks_exact(4); + let (chunks, trailing_blocks) = full_blocks.as_chunks::<4>(); - for chunk in &mut chunks { + for chunk in chunks { let blocks = [ u128::from_be_bytes(chunk[0]), u128::from_be_bytes(chunk[1]), @@ -289,7 +287,7 @@ fn ghash_update_padded_wide(mut acc: u128, h_polyval: u128, h_powers_rev: &[u128 acc = polyval::accumulate_4blocks(acc, h_polyval, h_powers_rev, &blocks); } - for block in chunks.remainder() { + for block in trailing_blocks { acc ^= u128::from_be_bytes(*block); acc = polyval::clmul128_reduce(acc, h_polyval); } @@ -380,6 +378,7 @@ fn encrypt_j0_tag(ek: &aes::Aes128EncKey, j0: &[u8; 16], acc: u128) -> [u8; TAG_ } #[cfg(feature = "diag")] +/// Exercise AES-128 counter-mode encryption and fold the fixed diagnostic output to one block. #[must_use] pub fn diag_aes128gcm_ctr32_be(cipher: &Aes128Gcm, nonce: &Nonce96, plaintext: &[u8; 44]) -> [u8; 16] { let (_, ctr_block) = make_j0_and_ctr(nonce); @@ -389,6 +388,7 @@ pub fn diag_aes128gcm_ctr32_be(cipher: &Aes128Gcm, nonce: &Nonce96, plaintext: & } #[cfg(feature = "diag")] +/// Return the AES-128-GCM GHASH accumulator for diagnostic backend comparison. #[unsafe(no_mangle)] #[inline(never)] #[must_use] @@ -412,6 +412,7 @@ pub fn diag_aes128gcm_ghash(cipher: &Aes128Gcm, aad: &[u8], ciphertext: &[u8]) - } #[cfg(feature = "diag")] +/// Encrypt a diagnostic GHASH accumulator into the final AES-128-GCM tag. #[must_use] pub fn diag_aes128gcm_tag_aes(cipher: &Aes128Gcm, nonce: &Nonce96, acc: &[u8; 16]) -> [u8; 16] { let (j0, _) = make_j0_and_ctr(nonce); @@ -505,9 +506,9 @@ unsafe fn ghash_update_padded_wide_aarch64( data: &[u8], ) -> u128 { let (full_blocks, remainder) = data.as_chunks::<16>(); - let mut chunks = full_blocks.chunks_exact(4); + let (chunks, trailing_blocks) = full_blocks.as_chunks::<4>(); - for chunk in &mut chunks { + for chunk in chunks { let blocks = [ u128::from_be_bytes(chunk[0]), u128::from_be_bytes(chunk[1]), @@ -520,7 +521,7 @@ unsafe fn ghash_update_padded_wide_aarch64( acc = unsafe { polyval::aarch64_aggregate_4blocks_inline(acc, h_powers_rev, &blocks) }; } - for block in chunks.remainder() { + for block in trailing_blocks { acc ^= u128::from_be_bytes(*block); // SAFETY: PMULL carryless multiply because: // 1. This function's caller must guarantee AES-CE/PMULL availability. @@ -549,9 +550,9 @@ unsafe fn ghash_update_padded_wide_aarch64( #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn ghash_update_padded_wide_ppc(mut acc: u128, h_polyval: u128, h_powers_rev: &[u128; 4], data: &[u8]) -> u128 { let (full_blocks, remainder) = data.as_chunks::<16>(); - let mut chunks = full_blocks.chunks_exact(4); + let (chunks, trailing_blocks) = full_blocks.as_chunks::<4>(); - for chunk in &mut chunks { + for chunk in chunks { let blocks = [ u128::from_be_bytes(chunk[0]), u128::from_be_bytes(chunk[1]), @@ -564,7 +565,7 @@ unsafe fn ghash_update_padded_wide_ppc(mut acc: u128, h_polyval: u128, h_powers_ acc = unsafe { polyval::ppc_aggregate_4blocks_inline(acc, h_powers_rev, &blocks) }; } - for block in chunks.remainder() { + for block in trailing_blocks { acc ^= u128::from_be_bytes(*block); // SAFETY: POWER8 carryless multiply because: // 1. This function's caller must guarantee POWER8 crypto availability. @@ -775,10 +776,6 @@ impl Aead for Aes128Gcm { // 2. `aad` is a valid byte slice; padding is handled inside the helper. let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }); - // SAFETY: fused intrinsic AArch64 AES-GCM sealing because: - // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE - // and PMULL. - // 2. The helper encrypts `buffer` in place and folds the resulting ciphertext into GHASH. let tables = aes::Aarch64GcmTables { h_polyval, h_powers_rev: &self.h_powers_rev, @@ -929,10 +926,6 @@ impl Aead for Aes128Gcm { // 2. `aad` is a valid byte slice; padding is handled inside the helper. let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }); - // SAFETY: fused intrinsic AArch64 AES-GCM open because: - // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE - // and PMULL. - // 2. The helper GHASHes ciphertext bytes before decrypting each chunk in place. let tables = aes::Aarch64GcmTables { h_polyval, h_powers_rev: &self.h_powers_rev, @@ -1076,10 +1069,13 @@ impl Drop for Aes128Gcm { #[cfg(test)] mod tests { - use alloc::{vec, vec::Vec}; + use alloc::vec; use super::*; - use crate::aead::expert::AeadWithNonce; + use crate::aead::{ + expert::AeadWithNonce, + test_vectors::{hex_vec, hex12, hex16}, + }; // NIST SP 800-38D Test Case 1: AES-128-GCM, empty plaintext, empty AAD. // Key: 00000000000000000000000000000000 @@ -1097,10 +1093,14 @@ mod tests { let expected_tag = hex16("58e2fccefa7e3061367f1d57a4e7455a"); let mut buf = vec![]; - let tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("NIST empty AES-128-GCM encryption must succeed"); assert_eq!(tag.0, expected_tag, "Tag mismatch on encrypt"); - cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &[], &mut buf, &tag) + .expect("NIST empty AES-128-GCM decryption must succeed"); } // NIST SP 800-38D Test Case 2: AES-128-GCM, 16-byte zero plaintext, empty AAD. @@ -1120,11 +1120,15 @@ mod tests { let expected_tag = hex16("ab6e47d42cec13bdf53a67b21257bddf"); let mut buf = vec![0u8; 16]; - let tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("NIST one-block AES-128-GCM encryption must succeed"); assert_eq!(buf, expected_ct, "Ciphertext mismatch"); assert_eq!(tag.0, expected_tag, "Tag mismatch"); - cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &[], &mut buf, &tag) + .expect("NIST one-block AES-128-GCM decryption must succeed"); assert_eq!(buf, vec![0u8; 16], "Plaintext mismatch after decrypt"); } @@ -1150,11 +1154,15 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("NIST multi-block AES-128-GCM encryption must succeed"); assert_eq!(buf, expected_ct, "Ciphertext mismatch"); assert_eq!(tag.0, expected_tag, "Tag mismatch"); - cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &[], &mut buf, &tag) + .expect("NIST multi-block AES-128-GCM decryption must succeed"); assert_eq!(buf, plaintext, "Plaintext mismatch after decrypt"); } @@ -1181,11 +1189,15 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut buf) + .expect("NIST AES-128-GCM encryption with AAD must succeed"); assert_eq!(buf, expected_ct, "Ciphertext mismatch"); assert_eq!(tag.0, expected_tag, "Tag mismatch"); - cipher.decrypt_in_place(&nonce, &aad, &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &aad, &mut buf, &tag) + .expect("NIST AES-128-GCM decryption with AAD must succeed"); assert_eq!(buf, plaintext, "Plaintext mismatch after decrypt"); } @@ -1197,11 +1209,15 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut buf = vec![0u8; 16]; - let mut tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let mut tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("AES-128-GCM test setup encryption must succeed"); tag.0[0] ^= 1; - let result = cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag), + Err(OpenError::verification()) + ); } /// Decryption with wrong AAD should fail. @@ -1210,14 +1226,17 @@ mod tests { let key = Aes128GcmKey::from_bytes(hex16("feffe9928665731c6d6a8f9467308308")); let nonce = Nonce96::from_bytes(hex12("cafebabefacedbaddecaf888")); let aad = hex_vec("feedfacedeadbeeffeedfacedeadbeefabaddad2"); - let plaintext = hex_vec("d9313225f88406e5a55909c5aff5269a"); let cipher = Aes128Gcm::new(&key); - let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut buf).unwrap(); + let mut buf = hex_vec("d9313225f88406e5a55909c5aff5269a"); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut buf) + .expect("AES-128-GCM test setup encryption must succeed"); - let result = cipher.decrypt_in_place(&nonce, b"wrong aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"wrong aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } /// Ciphertext tampering should fail verification. @@ -1228,11 +1247,15 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut buf = vec![0u8; 32]; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AES-128-GCM test setup encryption must succeed"); buf[0] ^= 1; - let result = cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } /// Detached encrypt/decrypt round-trip. @@ -1245,10 +1268,14 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut buf = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut buf) + .expect("AES-128-GCM detached encryption must succeed"); assert_ne!(&buf[..], &plaintext[..]); - cipher.decrypt_in_place(&nonce, aad, &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut buf, &tag) + .expect("AES-128-GCM detached decryption must succeed"); assert_eq!(&buf[..], &plaintext[..]); } @@ -1262,20 +1289,34 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, aad, plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, aad, plaintext, &mut out) + .expect("AES-128-GCM combined encryption must succeed"); let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, aad, &out, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, aad, &out, &mut pt_out) + .expect("AES-128-GCM combined decryption must succeed"); assert_eq!(&pt_out[..], &plaintext[..]); } /// `tag_from_slice` rejects wrong-length input. #[test] fn tag_from_slice_rejects_bad_length() { - assert!(Aes128Gcm::tag_from_slice(&[0u8; 15]).is_err()); - assert!(Aes128Gcm::tag_from_slice(&[0u8; 17]).is_err()); - assert!(Aes128Gcm::tag_from_slice(&[0u8; 0]).is_err()); - assert!(Aes128Gcm::tag_from_slice(&[0u8; 16]).is_ok()); + assert_eq!( + Aes128Gcm::tag_from_slice(&[0u8; 15]).expect_err("short AES-128-GCM tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes128Gcm::tag_from_slice(&[0u8; 17]).expect_err("long AES-128-GCM tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes128Gcm::tag_from_slice(&[]).expect_err("empty AES-128-GCM tag must be rejected"), + AeadBufferError::new() + ); + let tag = Aes128Gcm::tag_from_slice(&[0u8; 16]).expect("16-byte AES-128-GCM tag must be accepted"); + assert_eq!(tag.as_bytes(), &[0u8; 16]); } /// Decryption with wrong nonce must fail. @@ -1286,11 +1327,15 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut buf = *b"hello gcm"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AES-128-GCM test setup encryption must succeed"); let wrong_nonce = Nonce96::from_bytes([0x08u8; 12]); - let result = cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } /// On authentication failure, the output buffer must be zeroed. @@ -1302,42 +1347,18 @@ mod tests { let plaintext = *b"zero me on failure"; let mut buf = plaintext; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AES-128-GCM test setup encryption must succeed"); let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0xFF; let bad_tag = Aes128GcmTag::from_bytes(bad_tag); - let result = cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert!(buf.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } - - // --- Hex helpers --- - - fn hex16(hex: &str) -> [u8; 16] { - let mut out = [0u8; 16]; - for i in 0..16 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex12(hex: &str) -> [u8; 12] { - let mut out = [0u8; 12]; - for i in 0..12 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex_vec(hex: &str) -> Vec { - let mut out = Vec::with_capacity(hex.len() / 2); - let mut i = 0; - while i < hex.len() { - out.push(u8::from_str_radix(&hex[i..i + 2], 16).unwrap()); - i += 2; - } - out - } } diff --git a/src/aead/aes128gcmsiv.rs b/src/aead/aes128gcmsiv.rs index 2d9cceb4..9796f20f 100644 --- a/src/aead/aes128gcmsiv.rs +++ b/src/aead/aes128gcmsiv.rs @@ -1,15 +1,18 @@ -#![allow(clippy::indexing_slicing)] - //! AES-128-GCM-SIV public AEAD surface (RFC 8452). use core::fmt; #[cfg(target_arch = "x86_64")] use super::polyval::{accumulate_padded_x86, precompute_powers, precompute_powers_16}; -use super::{ - AeadBufferError, Nonce96, OpenError, SealError, aes, polyval, - targets::{AeadBackend, AeadPrimitive, select_backend}, -}; +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", +))] +use super::targets::{AeadBackend, AeadPrimitive, select_backend}; +use super::{AeadBufferError, Nonce96, OpenError, SealError, aes, polyval}; use crate::traits::{Aead, ct}; const KEY_SIZE: usize = 16; @@ -85,7 +88,13 @@ define_aead_tag_type!( /// timing claims are configuration- and release-evidence-bound; see `ct.toml`. pub struct Aes128GcmSiv { master_ek: aes::Aes128EncKey, - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + #[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ))] backend: AeadBackend, } @@ -219,12 +228,14 @@ fn compute_tag( } #[cfg(feature = "diag")] +/// Derive the per-nonce authentication and encryption keys for diagnostic comparison. #[must_use] pub fn diag_aes128gcmsiv_derive_keys(cipher: &Aes128GcmSiv, nonce: &Nonce96) -> ([u8; 16], [u8; 16]) { derive_keys(&cipher.master_ek, nonce) } #[cfg(feature = "diag")] +/// Return the AES-128-GCM-SIV POLYVAL digest before nonce and AES tag finalization. #[must_use] pub fn diag_aes128gcmsiv_polyval_digest(auth_key: &[u8; 16], aad: &[u8], plaintext: &[u8]) -> [u8; 16] { let mut pv = polyval::Polyval::new(auth_key); @@ -236,6 +247,7 @@ pub fn diag_aes128gcmsiv_polyval_digest(auth_key: &[u8; 16], aad: &[u8], plainte } #[cfg(feature = "diag")] +/// Encrypt one diagnostic tag block with a raw AES-128 key. #[must_use] pub fn diag_aes128gcmsiv_raw_tag_aes(enc_key: &[u8; 16], block: &[u8; 16]) -> [u8; 16] { let mut out = *block; @@ -253,6 +265,7 @@ pub fn diag_aes128gcmsiv_raw_tag_aes(enc_key: &[u8; 16], block: &[u8; 16]) -> [u } #[cfg(feature = "diag")] +/// Exercise AES-128 counter-mode encryption and fold the fixed diagnostic output to one block. #[must_use] pub fn diag_aes128gcmsiv_ctr32(enc_key: &[u8; 16], tag: &[u8; 16], plaintext: &[u8; 44]) -> [u8; 16] { let mut counter_block = *tag; @@ -404,7 +417,6 @@ fn expand_key_riscv_for_backend(key: &[u8; 16], backend: AeadBackend) -> aes::Ae match backend { AeadBackend::Riscv64VectorCrypto => aes::aes128_expand_key_riscv_vector(key), AeadBackend::Riscv64ScalarCrypto => aes::aes128_expand_key_riscv_scalar(key), - AeadBackend::Riscv64Vperm => aes::aes128_expand_key_riscv_vperm(key), AeadBackend::Portable => aes::aes128_expand_key_riscv_ttable(key), _ => aes::aes128_expand_key_riscv_ttable(key), } @@ -416,6 +428,13 @@ fn expand_message_key_riscv(enc_key: &[u8; 16], backend: AeadBackend) -> aes::Ae expand_key_riscv_for_backend(enc_key, backend) } +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", +))] #[inline] fn resolve_backend() -> AeadBackend { select_backend( @@ -431,7 +450,7 @@ fn riscv_polyval_backend(backend: AeadBackend) -> RiscvPolyvalBackend { match backend { AeadBackend::Riscv64VectorCrypto => RiscvPolyvalBackend::Vector, AeadBackend::Riscv64ScalarCrypto => RiscvPolyvalBackend::Scalar, - AeadBackend::Portable | AeadBackend::Riscv64Vperm => { + AeadBackend::Portable => { let caps = crate::platform::caps(); if caps.has(crate::platform::caps::riscv::ZBC) || caps.has(crate::platform::caps::riscv::ZBKC) { RiscvPolyvalBackend::Scalar @@ -559,6 +578,12 @@ fn compute_tag_wide( #[cfg(target_arch = "aarch64")] #[target_feature(enable = "aes,neon")] +/// Seal one message through the fused AArch64 AES-128-GCM-SIV path. +/// +/// # Safety +/// +/// The current CPU must support AArch64 AES, NEON, and PMULL. Callers must establish those +/// capabilities through validated backend selection before entering this function. unsafe fn encrypt_fused_aarch64( auth_key: &mut [u8; 16], enc_key_bytes: &mut [u8; 16], @@ -705,6 +730,12 @@ unsafe fn encrypt_fused_aarch64( #[cfg(target_arch = "aarch64")] #[target_feature(enable = "aes,neon")] +/// Open one message through the fused AArch64 AES-128-GCM-SIV path. +/// +/// # Safety +/// +/// The current CPU must support AArch64 AES, NEON, and PMULL. Callers must establish those +/// capabilities through validated backend selection before entering this function. unsafe fn decrypt_fused_aarch64( auth_key: &mut [u8; 16], enc_key_bytes: &mut [u8; 16], @@ -857,6 +888,11 @@ unsafe fn decrypt_fused_aarch64( // powerpc64 fused encrypt/decrypt (single #[target_feature] scope) +/// Encrypt with the fused POWER8 AES-128-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec, VSX, POWER8 vector, and POWER8 crypto. #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn encrypt_fused_ppc( @@ -986,6 +1022,11 @@ unsafe fn encrypt_fused_ppc( } } +/// Decrypt and authenticate with the fused POWER8 AES-128-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec, VSX, POWER8 vector, and POWER8 crypto. #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn decrypt_fused_ppc( @@ -1122,6 +1163,11 @@ unsafe fn decrypt_fused_ppc( // s390x fused encrypt/decrypt +/// XOR an AES-128 counter stream into `buffer` using s390x CPACF. +/// +/// # Safety +/// +/// The executing CPU must support MSA AES instructions. #[cfg(target_arch = "s390x")] unsafe fn s390x_ctr32_le_xor_raw(enc_key_bytes: &[u8; 16], counter_block: &mut [u8; 16], buffer: &mut [u8]) { let mut ctr = u32::from_le_bytes([counter_block[0], counter_block[1], counter_block[2], counter_block[3]]); @@ -1147,12 +1193,17 @@ unsafe fn s390x_ctr32_le_xor_raw(enc_key_bytes: &[u8; 16], counter_block: &mut [ let processed = aes::xor_keystream_tail(buffer, offset, &keystream, block_count); offset = offset.strict_add(processed); - ctr = ctr.wrapping_add(block_count as u32); + ctr = ctr.wrapping_add(u32::from(block_count.to_le_bytes()[0])); } counter_block[0..4].copy_from_slice(&ctr.to_le_bytes()); } +/// Encrypt with the fused s390x AES-128-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility and MSA AES instructions. #[cfg(target_arch = "s390x")] #[target_feature(enable = "vector")] unsafe fn encrypt_fused_s390x( @@ -1257,6 +1308,11 @@ unsafe fn encrypt_fused_s390x( } } +/// Decrypt and authenticate with the fused s390x AES-128-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility and MSA AES instructions. #[cfg(target_arch = "s390x")] #[target_feature(enable = "vector")] unsafe fn decrypt_fused_s390x( @@ -1376,6 +1432,13 @@ impl Aead for Aes128GcmSiv { type Tag = Aes128GcmSivTag; fn new(key: &Self::Key) -> Self { + #[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ))] let backend = resolve_backend(); Self { @@ -1383,6 +1446,13 @@ impl Aead for Aes128GcmSiv { master_ek: expand_key_riscv_for_backend(key.as_bytes(), backend), #[cfg(not(target_arch = "riscv64"))] master_ek: aes::aes128_expand_key(key.as_bytes()), + #[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ))] backend, } } @@ -1460,10 +1530,7 @@ impl Aead for Aes128GcmSiv { #[cfg(target_arch = "riscv64")] { match self.backend { - AeadBackend::Portable - | AeadBackend::Riscv64VectorCrypto - | AeadBackend::Riscv64ScalarCrypto - | AeadBackend::Riscv64Vperm => { + AeadBackend::Portable | AeadBackend::Riscv64VectorCrypto | AeadBackend::Riscv64ScalarCrypto => { let tag_bytes = encrypt_riscv(&self.master_ek, self.backend, nonce, aad, buffer); return Ok(Aes128GcmSivTag::from_bytes(tag_bytes)); } @@ -1556,10 +1623,7 @@ impl Aead for Aes128GcmSiv { #[cfg(target_arch = "riscv64")] { match self.backend { - AeadBackend::Portable - | AeadBackend::Riscv64VectorCrypto - | AeadBackend::Riscv64ScalarCrypto - | AeadBackend::Riscv64Vperm => { + AeadBackend::Portable | AeadBackend::Riscv64VectorCrypto | AeadBackend::Riscv64ScalarCrypto => { return decrypt_riscv(&self.master_ek, self.backend, nonce, aad, buffer, tag).map_err(OpenError::from); } _ => {} @@ -1590,10 +1654,13 @@ impl Aead for Aes128GcmSiv { #[cfg(test)] mod tests { - use alloc::{vec, vec::Vec}; + use alloc::vec; use super::*; - use crate::aead::expert::AeadWithNonce; + use crate::aead::{ + expert::AeadWithNonce, + test_vectors::{hex_vec, hex12, hex16}, + }; /// RFC 8452 Appendix C.1, test 1: empty plaintext, empty AAD. #[test] @@ -1604,11 +1671,15 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut out = vec![0u8; expected_ct_tag.len()]; - cipher.encrypt(&nonce, &[], &[], &mut out).unwrap(); + cipher + .encrypt(&nonce, &[], &[], &mut out) + .expect("RFC 8452 empty AES-128-GCM-SIV encryption must succeed"); assert_eq!(out, expected_ct_tag); let mut pt_out = vec![0u8; 0]; - cipher.decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 empty AES-128-GCM-SIV decryption must succeed"); assert!(pt_out.is_empty()); } @@ -1622,11 +1693,15 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &[], &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &[], &plaintext, &mut out) + .expect("RFC 8452 AES-128-GCM-SIV encryption must succeed"); assert_eq!(out, expected_ct_tag); let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 AES-128-GCM-SIV decryption must succeed"); assert_eq!(pt_out, plaintext); } @@ -1642,11 +1717,15 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut out) + .expect("RFC 8452 AES-128-GCM-SIV encryption with AAD must succeed"); assert_eq!(out, expected_ct_tag); let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &aad, &expected_ct_tag, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &aad, &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 AES-128-GCM-SIV decryption with AAD must succeed"); assert_eq!(pt_out, plaintext); } @@ -1659,22 +1738,16 @@ mod tests { let plaintext = hex_vec( "0200000000000000000000000000000003000000000000000000000000000000040000000000000000000000000000000500000000000000000000000000000000", ); - let expected_ct_tag = hex_vec( - "2f5c64059db55ee0fb847ed51300374651a8c75e07ecbef82c1ed1bf48bb1ff5c1cdf0a4a3aa6a17ce7022eafd1129b25aafdca1fa9706ce0d1e23df1ce04c4a8e10cb44ed62b9a3a39d61b8b3eba23586eb6df746a45dd8d6ce80e7f5d33d39", - ); - - // The expected vector above is illustrative; oracle tests at - // tests/aes128gcmsiv_oracle.rs are the authoritative byte-equivalence - // check. This unit test fixes the 4-block + AAD shape against the - // round-trip rather than a hand-transcribed vector. - let _ = expected_ct_tag; - let cipher = Aes128GcmSiv::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut out) + .expect("multi-block AES-128-GCM-SIV encryption must succeed"); let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &aad, &out, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &aad, &out, &mut pt_out) + .expect("multi-block AES-128-GCM-SIV decryption must succeed"); assert_eq!(pt_out, plaintext); } @@ -1688,8 +1761,10 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut pt_out = vec![0u8; 0]; - let result = cipher.decrypt(&nonce, &[], &bad_ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &[], &bad_ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// Decryption with wrong AAD should fail. @@ -1701,8 +1776,10 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut pt_out = vec![0u8; 8]; - let result = cipher.decrypt(&nonce, &[0x02], &ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &[0x02], &ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// Decryption with wrong nonce should fail. @@ -1715,8 +1792,10 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut pt_out = vec![0u8; 8]; let wrong_nonce = Nonce96::from_bytes(hex12("040000000000000000000000")); - let result = cipher.decrypt(&wrong_nonce, &aad, &ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&wrong_nonce, &aad, &ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// Ciphertext tampering should fail verification. @@ -1732,8 +1811,10 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut pt_out = vec![0u8; plaintext.len()]; - let result = cipher.decrypt(&nonce, &aad, &ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &aad, &ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// On authentication failure, the output buffer must be zeroed. @@ -1746,14 +1827,18 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut out) + .expect("AES-128-GCM-SIV test setup encryption must succeed"); let last = out.len().strict_sub(1); out[last] ^= 0xff; let mut pt_out = vec![0xffu8; plaintext.len()]; - let result = cipher.decrypt(&nonce, &aad, &out, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &aad, &out, &mut pt_out), + Err(OpenError::verification()) + ); assert!(pt_out.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } @@ -1768,57 +1853,51 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut buf) + .expect("AES-128-GCM-SIV detached encryption must succeed"); assert_ne!(buf, plaintext); - cipher.decrypt_in_place(&nonce, &aad, &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &aad, &mut buf, &tag) + .expect("AES-128-GCM-SIV detached decryption must succeed"); assert_eq!(buf, plaintext); } /// `tag_from_slice` rejects wrong-length input. #[test] fn aes128gcmsiv_tag_from_slice_rejects_bad_length() { - assert!(Aes128GcmSiv::tag_from_slice(&[0u8; 15]).is_err()); - assert!(Aes128GcmSiv::tag_from_slice(&[0u8; 17]).is_err()); - assert!(Aes128GcmSiv::tag_from_slice(&[0u8; 0]).is_err()); - assert!(Aes128GcmSiv::tag_from_slice(&[0u8; 16]).is_ok()); + assert_eq!( + Aes128GcmSiv::tag_from_slice(&[0u8; 15]).expect_err("short AES-128-GCM-SIV tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes128GcmSiv::tag_from_slice(&[0u8; 17]).expect_err("long AES-128-GCM-SIV tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes128GcmSiv::tag_from_slice(&[]).expect_err("empty AES-128-GCM-SIV tag must be rejected"), + AeadBufferError::new() + ); + let tag = Aes128GcmSiv::tag_from_slice(&[0u8; 16]).expect("16-byte AES-128-GCM-SIV tag must be accepted"); + assert_eq!(tag.as_bytes(), &[0u8; 16]); } #[test] #[cfg(target_pointer_width = "64")] fn aes128gcmsiv_input_limit_matches_rfc8452() { for len in [MAX_INPUT_LEN.strict_sub(1), MAX_INPUT_LEN] { - assert!(super::super::try_bounded_length_as_u64(len as usize, MAX_INPUT_LEN).is_ok()); - } - assert!(super::super::try_bounded_length_as_u64(MAX_INPUT_LEN.strict_add(1) as usize, MAX_INPUT_LEN).is_err()); - } - - // --- Hex helpers --- - - fn hex16(hex: &str) -> [u8; 16] { - let mut out = [0u8; 16]; - for i in 0..16 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex12(hex: &str) -> [u8; 12] { - let mut out = [0u8; 12]; - for i in 0..12 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex_vec(hex: &str) -> Vec { - let mut out = Vec::with_capacity(hex.len() / 2); - let mut i = 0; - while i < hex.len() { - out.push(u8::from_str_radix(&hex[i..i + 2], 16).unwrap()); - i += 2; - } - out + let platform_len = usize::try_from(len).expect("RFC 8452 input limit fits 64-bit usize"); + assert_eq!( + super::super::try_bounded_length_as_u64(platform_len, MAX_INPUT_LEN), + Ok(len) + ); + } + let too_large = usize::try_from(MAX_INPUT_LEN.strict_add(1)).expect("RFC 8452 input limit fits 64-bit usize"); + assert_eq!( + super::super::try_bounded_length_as_u64(too_large, MAX_INPUT_LEN), + Err(super::super::LengthOverflow) + ); } } diff --git a/src/aead/aes256gcm.rs b/src/aead/aes256gcm.rs index 2c806bc6..ba27f516 100644 --- a/src/aead/aes256gcm.rs +++ b/src/aead/aes256gcm.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! AES-256-GCM public AEAD surface (NIST SP 800-38D). use core::fmt; @@ -275,9 +273,9 @@ fn ghash_update_padded(mut acc: u128, h_polyval: u128, data: &[u8]) -> u128 { #[inline] fn ghash_update_padded_wide(mut acc: u128, h_polyval: u128, h_powers_rev: &[u128; 4], data: &[u8]) -> u128 { let (full_blocks, remainder) = data.as_chunks::<16>(); - let mut chunks = full_blocks.chunks_exact(4); + let (chunks, trailing_blocks) = full_blocks.as_chunks::<4>(); - for chunk in &mut chunks { + for chunk in chunks { let blocks = [ u128::from_be_bytes(chunk[0]), u128::from_be_bytes(chunk[1]), @@ -287,7 +285,7 @@ fn ghash_update_padded_wide(mut acc: u128, h_polyval: u128, h_powers_rev: &[u128 acc = polyval::accumulate_4blocks(acc, h_polyval, h_powers_rev, &blocks); } - for block in chunks.remainder() { + for block in trailing_blocks { acc ^= u128::from_be_bytes(*block); acc = polyval::clmul128_reduce(acc, h_polyval); } @@ -378,6 +376,7 @@ fn encrypt_j0_tag(ek: &aes::Aes256EncKey, j0: &[u8; 16], acc: u128) -> [u8; TAG_ } #[cfg(feature = "diag")] +/// Exercise AES-256 counter-mode encryption and fold the fixed diagnostic output to one block. #[must_use] pub fn diag_aes256gcm_ctr32_be(cipher: &Aes256Gcm, nonce: &Nonce96, plaintext: &[u8; 44]) -> [u8; 16] { let (_, ctr_block) = make_j0_and_ctr(nonce); @@ -387,6 +386,7 @@ pub fn diag_aes256gcm_ctr32_be(cipher: &Aes256Gcm, nonce: &Nonce96, plaintext: & } #[cfg(feature = "diag")] +/// Return the AES-256-GCM GHASH accumulator for diagnostic backend comparison. #[unsafe(no_mangle)] #[inline(never)] #[must_use] @@ -410,6 +410,7 @@ pub fn diag_aes256gcm_ghash(cipher: &Aes256Gcm, aad: &[u8], ciphertext: &[u8]) - } #[cfg(feature = "diag")] +/// Encrypt a diagnostic GHASH accumulator into the final AES-256-GCM tag. #[must_use] pub fn diag_aes256gcm_tag_aes(cipher: &Aes256Gcm, nonce: &Nonce96, acc: &[u8; 16]) -> [u8; 16] { let (j0, _) = make_j0_and_ctr(nonce); @@ -506,9 +507,9 @@ unsafe fn ghash_update_padded_wide_aarch64( data: &[u8], ) -> u128 { let (full_blocks, remainder) = data.as_chunks::<16>(); - let mut chunks = full_blocks.chunks_exact(4); + let (chunks, trailing_blocks) = full_blocks.as_chunks::<4>(); - for chunk in &mut chunks { + for chunk in chunks { let blocks = [ u128::from_be_bytes(chunk[0]), u128::from_be_bytes(chunk[1]), @@ -521,7 +522,7 @@ unsafe fn ghash_update_padded_wide_aarch64( acc = unsafe { polyval::aarch64_aggregate_4blocks_inline(acc, h_powers_rev, &blocks) }; } - for block in chunks.remainder() { + for block in trailing_blocks { acc ^= u128::from_be_bytes(*block); // SAFETY: PMULL carryless multiply because: // 1. This function's caller must guarantee AES-CE/PMULL availability. @@ -550,9 +551,9 @@ unsafe fn ghash_update_padded_wide_aarch64( #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn ghash_update_padded_wide_ppc(mut acc: u128, h_polyval: u128, h_powers_rev: &[u128; 4], data: &[u8]) -> u128 { let (full_blocks, remainder) = data.as_chunks::<16>(); - let mut chunks = full_blocks.chunks_exact(4); + let (chunks, trailing_blocks) = full_blocks.as_chunks::<4>(); - for chunk in &mut chunks { + for chunk in chunks { let blocks = [ u128::from_be_bytes(chunk[0]), u128::from_be_bytes(chunk[1]), @@ -565,7 +566,7 @@ unsafe fn ghash_update_padded_wide_ppc(mut acc: u128, h_polyval: u128, h_powers_ acc = unsafe { polyval::ppc_aggregate_4blocks_inline(acc, h_powers_rev, &blocks) }; } - for block in chunks.remainder() { + for block in trailing_blocks { acc ^= u128::from_be_bytes(*block); // SAFETY: POWER8 carryless multiply because: // 1. This function's caller must guarantee POWER8 crypto availability. @@ -778,10 +779,6 @@ impl Aead for Aes256Gcm { // 2. `aad` is a valid byte slice; padding is handled inside the helper. let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }); - // SAFETY: fused intrinsic AArch64 AES-GCM sealing because: - // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE - // and PMULL. - // 2. The helper encrypts `buffer` in place and folds the resulting ciphertext into GHASH. let tables = aes::Aarch64GcmTables { h_polyval, h_powers_rev: &self.h_powers_rev, @@ -932,10 +929,6 @@ impl Aead for Aes256Gcm { // 2. `aad` is a valid byte slice; padding is handled inside the helper. let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }); - // SAFETY: fused intrinsic AArch64 AES-GCM open because: - // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE - // and PMULL. - // 2. The helper GHASHes ciphertext bytes before decrypting each chunk in place. let tables = aes::Aarch64GcmTables { h_polyval, h_powers_rev: &self.h_powers_rev, @@ -1079,10 +1072,13 @@ impl Drop for Aes256Gcm { #[cfg(test)] mod tests { - use alloc::{vec, vec::Vec}; + use alloc::vec; use super::*; - use crate::aead::expert::AeadWithNonce; + use crate::aead::{ + expert::AeadWithNonce, + test_vectors::{hex_vec, hex12, hex16, hex32}, + }; // NIST SP 800-38D Test Case 13: AES-256-GCM, empty plaintext, empty AAD. // Key: 0000...00 (32 bytes) @@ -1101,11 +1097,15 @@ mod tests { // Encrypt. let mut buf = vec![]; - let tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("NIST empty AES-256-GCM encryption must succeed"); assert_eq!(tag.0, expected_tag, "Tag mismatch on encrypt"); // Decrypt. - cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &[], &mut buf, &tag) + .expect("NIST empty AES-256-GCM decryption must succeed"); } // NIST SP 800-38D Test Case 14: AES-256-GCM, 16-byte plaintext, empty AAD. @@ -1126,12 +1126,16 @@ mod tests { // Encrypt. let mut buf = vec![0u8; 16]; - let tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("NIST one-block AES-256-GCM encryption must succeed"); assert_eq!(buf, expected_ct, "Ciphertext mismatch"); assert_eq!(tag.0, expected_tag, "Tag mismatch"); // Decrypt. - cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &[], &mut buf, &tag) + .expect("NIST one-block AES-256-GCM decryption must succeed"); assert_eq!(buf, vec![0u8; 16], "Plaintext mismatch after decrypt"); } @@ -1160,12 +1164,16 @@ mod tests { // Encrypt. let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("NIST multi-block AES-256-GCM encryption must succeed"); assert_eq!(buf, expected_ct, "Ciphertext mismatch"); assert_eq!(tag.0, expected_tag, "Tag mismatch"); // Decrypt. - cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &[], &mut buf, &tag) + .expect("NIST multi-block AES-256-GCM decryption must succeed"); assert_eq!(buf, plaintext, "Plaintext mismatch after decrypt"); } @@ -1195,12 +1203,16 @@ mod tests { // Encrypt. let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut buf) + .expect("NIST AES-256-GCM encryption with AAD must succeed"); assert_eq!(buf, expected_ct, "Ciphertext mismatch"); assert_eq!(tag.0, expected_tag, "Tag mismatch"); // Decrypt. - cipher.decrypt_in_place(&nonce, &aad, &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &aad, &mut buf, &tag) + .expect("NIST AES-256-GCM decryption with AAD must succeed"); assert_eq!(buf, plaintext, "Plaintext mismatch after decrypt"); } @@ -1212,11 +1224,15 @@ mod tests { let cipher = Aes256Gcm::new(&key); let mut buf = vec![0u8; 16]; - let mut tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let mut tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("AES-256-GCM test setup encryption must succeed"); tag.0[0] ^= 1; - let result = cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag), + Err(OpenError::verification()) + ); } /// Decryption with wrong AAD should fail. @@ -1227,15 +1243,18 @@ mod tests { )); let nonce = Nonce96::from_bytes(hex12("cafebabefacedbaddecaf888")); let aad = hex_vec("feedfacedeadbeeffeedfacedeadbeefabaddad2"); - let plaintext = hex_vec("d9313225f88406e5a55909c5aff5269a"); let cipher = Aes256Gcm::new(&key); - let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut buf).unwrap(); + let mut buf = hex_vec("d9313225f88406e5a55909c5aff5269a"); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut buf) + .expect("AES-256-GCM test setup encryption must succeed"); // Wrong AAD. - let result = cipher.decrypt_in_place(&nonce, b"wrong aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"wrong aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } /// Ciphertext tampering should fail verification. @@ -1246,11 +1265,15 @@ mod tests { let cipher = Aes256Gcm::new(&key); let mut buf = vec![0u8; 32]; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AES-256-GCM test setup encryption must succeed"); buf[0] ^= 1; - let result = cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } /// Detached encrypt/decrypt round-trip. @@ -1263,10 +1286,14 @@ mod tests { let cipher = Aes256Gcm::new(&key); let mut buf = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut buf) + .expect("AES-256-GCM detached encryption must succeed"); assert_ne!(&buf[..], &plaintext[..]); - cipher.decrypt_in_place(&nonce, aad, &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut buf, &tag) + .expect("AES-256-GCM detached decryption must succeed"); assert_eq!(&buf[..], &plaintext[..]); } @@ -1280,56 +1307,34 @@ mod tests { let cipher = Aes256Gcm::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, aad, plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, aad, plaintext, &mut out) + .expect("AES-256-GCM combined encryption must succeed"); let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, aad, &out, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, aad, &out, &mut pt_out) + .expect("AES-256-GCM combined decryption must succeed"); assert_eq!(&pt_out[..], &plaintext[..]); } /// `tag_from_slice` rejects wrong-length input. #[test] fn tag_from_slice_rejects_bad_length() { - assert!(Aes256Gcm::tag_from_slice(&[0u8; 15]).is_err()); - assert!(Aes256Gcm::tag_from_slice(&[0u8; 17]).is_err()); - assert!(Aes256Gcm::tag_from_slice(&[0u8; 0]).is_err()); - assert!(Aes256Gcm::tag_from_slice(&[0u8; 16]).is_ok()); - } - - // --- Hex helpers --- - - fn hex16(hex: &str) -> [u8; 16] { - let mut out = [0u8; 16]; - for i in 0..16 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex32(hex: &str) -> [u8; 32] { - let mut out = [0u8; 32]; - for i in 0..32 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex12(hex: &str) -> [u8; 12] { - let mut out = [0u8; 12]; - for i in 0..12 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex_vec(hex: &str) -> Vec { - let mut out = Vec::with_capacity(hex.len() / 2); - let mut i = 0; - while i < hex.len() { - out.push(u8::from_str_radix(&hex[i..i + 2], 16).unwrap()); - i += 2; - } - out + assert_eq!( + Aes256Gcm::tag_from_slice(&[0u8; 15]).expect_err("short AES-256-GCM tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes256Gcm::tag_from_slice(&[0u8; 17]).expect_err("long AES-256-GCM tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes256Gcm::tag_from_slice(&[]).expect_err("empty AES-256-GCM tag must be rejected"), + AeadBufferError::new() + ); + let tag = Aes256Gcm::tag_from_slice(&[0u8; 16]).expect("16-byte AES-256-GCM tag must be accepted"); + assert_eq!(tag.as_bytes(), &[0u8; 16]); } /// Decryption with wrong nonce must fail. @@ -1340,11 +1345,15 @@ mod tests { let cipher = Aes256Gcm::new(&key); let mut buf = *b"hello gcm"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AES-256-GCM test setup encryption must succeed"); let wrong_nonce = Nonce96::from_bytes([0x08u8; 12]); - let result = cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } /// On authentication failure, the output buffer must be zeroed. @@ -1356,15 +1365,19 @@ mod tests { let plaintext = *b"zero me on failure"; let mut buf = plaintext; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AES-256-GCM test setup encryption must succeed"); // Corrupt the tag. let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0xFF; let bad_tag = Aes256GcmTag::from_bytes(bad_tag); - let result = cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert!(buf.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } } diff --git a/src/aead/aes256gcmsiv.rs b/src/aead/aes256gcmsiv.rs index 57824a7f..14376a2e 100644 --- a/src/aead/aes256gcmsiv.rs +++ b/src/aead/aes256gcmsiv.rs @@ -1,15 +1,18 @@ -#![allow(clippy::indexing_slicing)] - //! AES-256-GCM-SIV public AEAD surface (RFC 8452). use core::fmt; #[cfg(target_arch = "x86_64")] use super::polyval::{accumulate_padded_x86, precompute_powers, precompute_powers_16}; -use super::{ - AeadBufferError, Nonce96, OpenError, SealError, aes, polyval, - targets::{AeadBackend, AeadPrimitive, select_backend}, -}; +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", +))] +use super::targets::{AeadBackend, AeadPrimitive, select_backend}; +use super::{AeadBufferError, Nonce96, OpenError, SealError, aes, polyval}; use crate::traits::{Aead, ct}; const KEY_SIZE: usize = 32; @@ -84,7 +87,13 @@ define_aead_tag_type!( /// are configuration- and release-evidence-bound; see `ct.toml`. pub struct Aes256GcmSiv { master_ek: aes::Aes256EncKey, - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + #[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ))] backend: AeadBackend, } @@ -227,12 +236,14 @@ fn compute_tag( } #[cfg(feature = "diag")] +/// Derive the per-nonce authentication and encryption keys for diagnostic comparison. #[must_use] pub fn diag_aes256gcmsiv_derive_keys(cipher: &Aes256GcmSiv, nonce: &Nonce96) -> ([u8; 16], [u8; 32]) { derive_keys(&cipher.master_ek, nonce) } #[cfg(feature = "diag")] +/// Encrypt one diagnostic tag block with a raw AES-256 key. #[must_use] pub fn diag_aes256gcmsiv_raw_tag_aes(enc_key: &[u8; 32], block: &[u8; 16]) -> [u8; 16] { let mut out = *block; @@ -250,6 +261,7 @@ pub fn diag_aes256gcmsiv_raw_tag_aes(enc_key: &[u8; 32], block: &[u8; 16]) -> [u } #[cfg(feature = "diag")] +/// Exercise AES-256 counter-mode encryption and fold the fixed diagnostic output to one block. #[must_use] pub fn diag_aes256gcmsiv_ctr32(enc_key: &[u8; 32], tag: &[u8; 16], plaintext: &[u8; 44]) -> [u8; 16] { let mut counter_block = *tag; @@ -401,7 +413,6 @@ fn expand_key_riscv_for_backend(key: &[u8; 32], backend: AeadBackend) -> aes::Ae match backend { AeadBackend::Riscv64VectorCrypto => aes::aes256_expand_key_riscv_vector(key), AeadBackend::Riscv64ScalarCrypto => aes::aes256_expand_key_riscv_scalar(key), - AeadBackend::Riscv64Vperm => aes::aes256_expand_key_riscv_vperm(key), AeadBackend::Portable => aes::aes256_expand_key_riscv_ttable(key), _ => aes::aes256_expand_key_riscv_ttable(key), } @@ -413,6 +424,13 @@ fn expand_message_key_riscv(enc_key: &[u8; 32], backend: AeadBackend) -> aes::Ae expand_key_riscv_for_backend(enc_key, backend) } +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", +))] #[inline] fn resolve_backend() -> AeadBackend { select_backend( @@ -428,7 +446,7 @@ fn riscv_polyval_backend(backend: AeadBackend) -> RiscvPolyvalBackend { match backend { AeadBackend::Riscv64VectorCrypto => RiscvPolyvalBackend::Vector, AeadBackend::Riscv64ScalarCrypto => RiscvPolyvalBackend::Scalar, - AeadBackend::Portable | AeadBackend::Riscv64Vperm => { + AeadBackend::Portable => { let caps = crate::platform::caps(); if caps.has(crate::platform::caps::riscv::ZBC) || caps.has(crate::platform::caps::riscv::ZBKC) { RiscvPolyvalBackend::Scalar @@ -569,6 +587,12 @@ fn compute_tag_wide( #[cfg(target_arch = "aarch64")] #[target_feature(enable = "aes,neon")] +/// Seal one message through the fused AArch64 AES-256-GCM-SIV path. +/// +/// # Safety +/// +/// The current CPU must support AArch64 AES, NEON, and PMULL. Callers must establish those +/// capabilities through validated backend selection before entering this function. unsafe fn encrypt_fused_aarch64( auth_key: &mut [u8; 16], enc_key_bytes: &mut [u8; 32], @@ -725,6 +749,12 @@ unsafe fn encrypt_fused_aarch64( #[cfg(target_arch = "aarch64")] #[target_feature(enable = "aes,neon")] +/// Open one message through the fused AArch64 AES-256-GCM-SIV path. +/// +/// # Safety +/// +/// The current CPU must support AArch64 AES, NEON, and PMULL. Callers must establish those +/// capabilities through validated backend selection before entering this function. unsafe fn decrypt_fused_aarch64( auth_key: &mut [u8; 16], enc_key_bytes: &mut [u8; 32], @@ -886,6 +916,11 @@ unsafe fn decrypt_fused_aarch64( // powerpc64 fused encrypt/decrypt (single #[target_feature] scope) +/// Encrypt with the fused POWER8 AES-256-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec, VSX, POWER8 vector, and POWER8 crypto. #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn encrypt_fused_ppc( @@ -1025,6 +1060,11 @@ unsafe fn encrypt_fused_ppc( } } +/// Decrypt and authenticate with the fused POWER8 AES-256-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec, VSX, POWER8 vector, and POWER8 crypto. #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn decrypt_fused_ppc( @@ -1171,6 +1211,11 @@ unsafe fn decrypt_fused_ppc( // s390x fused encrypt/decrypt (#[target_feature(enable = "vector")] for POLYVAL) +/// XOR an AES-256 counter stream into `buffer` using s390x CPACF. +/// +/// # Safety +/// +/// The executing CPU must support MSA AES instructions. #[cfg(target_arch = "s390x")] unsafe fn s390x_ctr32_le_xor_raw(enc_key_bytes: &[u8; 32], counter_block: &mut [u8; 16], buffer: &mut [u8]) { let mut ctr = u32::from_le_bytes([counter_block[0], counter_block[1], counter_block[2], counter_block[3]]); @@ -1196,12 +1241,17 @@ unsafe fn s390x_ctr32_le_xor_raw(enc_key_bytes: &[u8; 32], counter_block: &mut [ let processed = aes::xor_keystream_tail(buffer, offset, &keystream, block_count); offset = offset.strict_add(processed); - ctr = ctr.wrapping_add(block_count as u32); + ctr = ctr.wrapping_add(u32::from(block_count.to_le_bytes()[0])); } counter_block[0..4].copy_from_slice(&ctr.to_le_bytes()); } +/// Encrypt with the fused s390x AES-256-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility and MSA AES instructions. #[cfg(target_arch = "s390x")] #[target_feature(enable = "vector")] unsafe fn encrypt_fused_s390x( @@ -1315,6 +1365,11 @@ unsafe fn encrypt_fused_s390x( } } +/// Decrypt and authenticate with the fused s390x AES-256-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility and MSA AES instructions. #[cfg(target_arch = "s390x")] #[target_feature(enable = "vector")] unsafe fn decrypt_fused_s390x( @@ -1443,6 +1498,13 @@ impl Aead for Aes256GcmSiv { type Tag = Aes256GcmSivTag; fn new(key: &Self::Key) -> Self { + #[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ))] let backend = resolve_backend(); Self { @@ -1450,6 +1512,13 @@ impl Aead for Aes256GcmSiv { master_ek: expand_key_riscv_for_backend(key.as_bytes(), backend), #[cfg(not(target_arch = "riscv64"))] master_ek: aes::aes256_expand_key(key.as_bytes()), + #[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ))] backend, } } @@ -1527,10 +1596,7 @@ impl Aead for Aes256GcmSiv { #[cfg(target_arch = "riscv64")] { match self.backend { - AeadBackend::Portable - | AeadBackend::Riscv64VectorCrypto - | AeadBackend::Riscv64ScalarCrypto - | AeadBackend::Riscv64Vperm => { + AeadBackend::Portable | AeadBackend::Riscv64VectorCrypto | AeadBackend::Riscv64ScalarCrypto => { let tag_bytes = encrypt_riscv(&self.master_ek, self.backend, nonce, aad, buffer); return Ok(Aes256GcmSivTag::from_bytes(tag_bytes)); } @@ -1623,10 +1689,7 @@ impl Aead for Aes256GcmSiv { #[cfg(target_arch = "riscv64")] { match self.backend { - AeadBackend::Portable - | AeadBackend::Riscv64VectorCrypto - | AeadBackend::Riscv64ScalarCrypto - | AeadBackend::Riscv64Vperm => { + AeadBackend::Portable | AeadBackend::Riscv64VectorCrypto | AeadBackend::Riscv64ScalarCrypto => { return decrypt_riscv(&self.master_ek, self.backend, nonce, aad, buffer, tag).map_err(OpenError::from); } _ => {} @@ -1657,10 +1720,13 @@ impl Aead for Aes256GcmSiv { #[cfg(test)] mod tests { - use alloc::{vec, vec::Vec}; + use alloc::vec; use super::*; - use crate::aead::expert::AeadWithNonce; + use crate::aead::{ + expert::AeadWithNonce, + test_vectors::{hex_vec, hex12, hex32}, + }; /// RFC 8452 Appendix C.2, test case 1: empty plaintext, empty AAD. #[test] @@ -1673,12 +1739,16 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut out = vec![0u8; expected_ct_tag.len()]; - cipher.encrypt(&nonce, &[], &[], &mut out).unwrap(); + cipher + .encrypt(&nonce, &[], &[], &mut out) + .expect("RFC 8452 empty AES-256-GCM-SIV encryption must succeed"); assert_eq!(out, expected_ct_tag); // Decrypt. let mut pt_out = vec![0u8; 0]; - cipher.decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 empty AES-256-GCM-SIV decryption must succeed"); assert!(pt_out.is_empty()); } @@ -1697,12 +1767,16 @@ mod tests { // Encrypt. let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut out) + .expect("RFC 8452 AES-256-GCM-SIV encryption with AAD must succeed"); assert_eq!(out, expected_ct_tag); // Decrypt. let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &aad, &expected_ct_tag, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &aad, &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 AES-256-GCM-SIV decryption with AAD must succeed"); assert_eq!(pt_out, plaintext); } @@ -1720,11 +1794,15 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut out) + .expect("RFC 8452 long AES-256-GCM-SIV encryption must succeed"); assert_eq!(out, expected_ct_tag); let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &aad, &expected_ct_tag, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &aad, &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 long AES-256-GCM-SIV decryption must succeed"); assert_eq!(pt_out, plaintext); } @@ -1741,8 +1819,10 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut pt_out = vec![0u8; 0]; - let result = cipher.decrypt(&nonce, &[], &bad_ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &[], &bad_ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// Decryption with wrong AAD should fail. @@ -1757,8 +1837,10 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut pt_out = vec![0u8; 8]; // plaintext was 8 bytes // Wrong AAD: 0x02 instead of 0x01. - let result = cipher.decrypt(&nonce, &[0x02], &ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &[0x02], &ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// Decryption with wrong nonce should fail. @@ -1774,8 +1856,10 @@ mod tests { let mut pt_out = vec![0u8; 8]; // plaintext was 8 bytes // Wrong nonce: 0x04 instead of 0x03. let wrong_nonce = Nonce96::from_bytes(hex12("040000000000000000000000")); - let result = cipher.decrypt(&wrong_nonce, &aad, &ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&wrong_nonce, &aad, &ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// Ciphertext body tampering should fail verification. @@ -1794,8 +1878,10 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut pt_out = vec![0u8; plaintext.len()]; - let result = cipher.decrypt(&nonce, &aad, &ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &aad, &ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// On authentication failure, the output buffer must be zeroed. @@ -1810,15 +1896,19 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut out) + .expect("AES-256-GCM-SIV test setup encryption must succeed"); // Corrupt the tag. let last = out.len().strict_sub(1); out[last] ^= 0xff; let mut pt_out = vec![0xffu8; plaintext.len()]; // fill with non-zero - let result = cipher.decrypt(&nonce, &aad, &out, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &aad, &out, &mut pt_out), + Err(OpenError::verification()) + ); // Buffer must be zeroed even though decryption was attempted. assert!(pt_out.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } @@ -1836,31 +1926,53 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut buf) + .expect("AES-256-GCM-SIV detached encryption must succeed"); // buf is now ciphertext, tag is separate. assert_ne!(buf, plaintext); - cipher.decrypt_in_place(&nonce, &aad, &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &aad, &mut buf, &tag) + .expect("AES-256-GCM-SIV detached decryption must succeed"); assert_eq!(buf, plaintext); } /// `tag_from_slice` rejects wrong-length input. #[test] fn aes256gcmsiv_tag_from_slice_rejects_bad_length() { - assert!(Aes256GcmSiv::tag_from_slice(&[0u8; 15]).is_err()); - assert!(Aes256GcmSiv::tag_from_slice(&[0u8; 17]).is_err()); - assert!(Aes256GcmSiv::tag_from_slice(&[0u8; 0]).is_err()); - assert!(Aes256GcmSiv::tag_from_slice(&[0u8; 16]).is_ok()); + assert_eq!( + Aes256GcmSiv::tag_from_slice(&[0u8; 15]).expect_err("short AES-256-GCM-SIV tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes256GcmSiv::tag_from_slice(&[0u8; 17]).expect_err("long AES-256-GCM-SIV tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes256GcmSiv::tag_from_slice(&[]).expect_err("empty AES-256-GCM-SIV tag must be rejected"), + AeadBufferError::new() + ); + let tag = Aes256GcmSiv::tag_from_slice(&[0u8; 16]).expect("16-byte AES-256-GCM-SIV tag must be accepted"); + assert_eq!(tag.as_bytes(), &[0u8; 16]); } #[test] #[cfg(target_pointer_width = "64")] fn aes256gcmsiv_input_limit_matches_rfc8452() { for len in [MAX_INPUT_LEN.strict_sub(1), MAX_INPUT_LEN] { - assert!(super::super::try_bounded_length_as_u64(len as usize, MAX_INPUT_LEN).is_ok()); - } - assert!(super::super::try_bounded_length_as_u64(MAX_INPUT_LEN.strict_add(1) as usize, MAX_INPUT_LEN).is_err()); + let platform_len = usize::try_from(len).expect("RFC 8452 input limit fits 64-bit usize"); + assert_eq!( + super::super::try_bounded_length_as_u64(platform_len, MAX_INPUT_LEN), + Ok(len) + ); + } + let too_large = usize::try_from(MAX_INPUT_LEN.strict_add(1)).expect("RFC 8452 input limit fits 64-bit usize"); + assert_eq!( + super::super::try_bounded_length_as_u64(too_large, MAX_INPUT_LEN), + Err(super::super::LengthOverflow) + ); } /// RFC 8452 Appendix C.2 vector with a different key (empty PT+AAD). @@ -1875,38 +1987,14 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut out = vec![0u8; expected_ct_tag.len()]; - cipher.encrypt(&nonce, &[], &[], &mut out).unwrap(); + cipher + .encrypt(&nonce, &[], &[], &mut out) + .expect("RFC 8452 alternate-key AES-256-GCM-SIV encryption must succeed"); assert_eq!(out, expected_ct_tag); let mut pt_out = vec![0u8; 0]; - cipher.decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out).unwrap(); - } - - // --- Hex helpers --- - - fn hex32(hex: &str) -> [u8; 32] { - let mut out = [0u8; 32]; - for i in 0..32 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex12(hex: &str) -> [u8; 12] { - let mut out = [0u8; 12]; - for i in 0..12 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex_vec(hex: &str) -> Vec { - let mut out = Vec::with_capacity(hex.len() / 2); - let mut i = 0; - while i < hex.len() { - out.push(u8::from_str_radix(&hex[i..i + 2], 16).unwrap()); - i += 2; - } - out + cipher + .decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 alternate-key AES-256-GCM-SIV decryption must succeed"); } } diff --git a/src/aead/aes_round.rs b/src/aead/aes_round.rs index 09f48810..73538fd8 100644 --- a/src/aead/aes_round.rs +++ b/src/aead/aes_round.rs @@ -23,7 +23,7 @@ const BLOCK_SIZE: usize = 16; /// Input transform, low nibble: maps AES GF(2^8) basis to tower-field basis. #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_IPT_LO: [u8; 16] = [ 0x00, 0x70, 0x2A, 0x5A, 0x98, 0xE8, 0xB2, 0xC2, 0x08, 0x78, 0x22, 0x52, 0x90, 0xE0, 0xBA, 0xCA, @@ -31,7 +31,7 @@ pub(crate) const VPERM_IPT_LO: [u8; 16] = [ /// Input transform, high nibble: maps AES GF(2^8) basis to tower-field basis. #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_IPT_HI: [u8; 16] = [ 0x00, 0x4D, 0x7C, 0x31, 0x7D, 0x30, 0x01, 0x4C, 0x81, 0xCC, 0xFD, 0xB0, 0xFC, 0xB1, 0x80, 0xCD, @@ -39,7 +39,7 @@ pub(crate) const VPERM_IPT_HI: [u8; 16] = [ /// GF(2^4) inverse table, low nibble. Entry 0 is `0x80` (infinity sentinel). #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_INV_LO: [u8; 16] = [ 0x80, 0x01, 0x08, 0x0D, 0x0F, 0x06, 0x05, 0x0E, 0x02, 0x0C, 0x0B, 0x0A, 0x09, 0x03, 0x07, 0x04, @@ -47,7 +47,7 @@ pub(crate) const VPERM_INV_LO: [u8; 16] = [ /// GF(2^4) inverse table, high nibble. Entry 0 is `0x80` (infinity sentinel). #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_INV_HI: [u8; 16] = [ 0x80, 0x07, 0x0B, 0x0F, 0x06, 0x0A, 0x04, 0x01, 0x09, 0x08, 0x05, 0x02, 0x0C, 0x0E, 0x0D, 0x03, @@ -56,7 +56,7 @@ pub(crate) const VPERM_INV_HI: [u8; 16] = [ /// SubBytes output table, upper component (inverse-isomorphism + AES affine, /// no MixColumns). `sbou[io] ^ sbot[jo] = AES_SBOX[x] ^ 0x63`. #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_SBOU: [u8; 16] = [ 0x00, 0xC7, 0xBD, 0x6F, 0x17, 0x6D, 0xD2, 0xD0, 0x78, 0xA8, 0x02, 0xC5, 0x7A, 0xBF, 0xAA, 0x15, @@ -65,7 +65,7 @@ pub(crate) const VPERM_SBOU: [u8; 16] = [ /// SubBytes output table, lower component (inverse-isomorphism + AES affine, /// no MixColumns). `sbou[io] ^ sbot[jo] = AES_SBOX[x] ^ 0x63`. #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_SBOT: [u8; 16] = [ 0x00, 0x6A, 0xBB, 0x5F, 0xA5, 0x74, 0xE4, 0xCF, 0xFA, 0x35, 0x2B, 0x41, 0xD1, 0x90, 0x1E, 0x8E, @@ -73,18 +73,18 @@ pub(crate) const VPERM_SBOT: [u8; 16] = [ /// AES ShiftRows permutation: output byte `i` reads from input byte `SR[i]`. #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_SR: [u8; 16] = [ 0x00, 0x05, 0x0A, 0x0F, 0x04, 0x09, 0x0E, 0x03, 0x08, 0x0D, 0x02, 0x07, 0x0C, 0x01, 0x06, 0x0B, ]; /// AES affine constant: Hamburg vperm S-box omits this; XOR after SubBytes. -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const AES_AFFINE: u8 = 0x63; /// AES MixColumns reduction constant: `xtime(b) = (b<<1) ^ ((b>>7) * 0x1B)`. -#[cfg(any(target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(target_arch = "s390x")] pub(crate) const XTIME_REDUCE: u8 = 0x1B; /// Nibble mask: extract low 4 bits of each byte. @@ -94,7 +94,7 @@ pub(crate) const NIBBLE_MASK: u8 = 0x0F; /// MixColumns column-rotate-by-1 permutation: rotates each 4-byte column by 1. /// `[b1,b2,b3,b0, b5,b6,b7,b4, b9,b10,b11,b8, b13,b14,b15,b12]` #[rustfmt::skip] -#[cfg(any(target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(target_arch = "s390x")] pub(crate) const MC_ROT1: [u8; 16] = [ 0x01, 0x02, 0x03, 0x00, 0x05, 0x06, 0x07, 0x04, 0x09, 0x0A, 0x0B, 0x08, 0x0D, 0x0E, 0x0F, 0x0C, @@ -103,7 +103,7 @@ pub(crate) const MC_ROT1: [u8; 16] = [ /// MixColumns column-rotate-by-2 permutation: rotates each 4-byte column by 2. /// `[b2,b3,b0,b1, b6,b7,b4,b5, b10,b11,b8,b9, b14,b15,b12,b13]` #[rustfmt::skip] -#[cfg(any(target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(target_arch = "s390x")] pub(crate) const MC_ROT2: [u8; 16] = [ 0x02, 0x03, 0x00, 0x01, 0x06, 0x07, 0x04, 0x05, 0x0A, 0x0B, 0x08, 0x09, 0x0E, 0x0F, 0x0C, 0x0D, @@ -140,7 +140,9 @@ const fn gf256_mul(a: u8, b: u8) -> u8 { prod ^= (prod >> 9).wrapping_mul(0x11b << 1); prod ^= (prod >> 8).wrapping_mul(0x11b); - prod as u8 + let [low, high] = prod.to_le_bytes(); + debug_assert!(high == 0); + low } #[cfg(any( @@ -194,7 +196,7 @@ const fn sbox(x: u8) -> u8 { ))] #[inline(always)] const fn col_byte(col: u32, row: usize) -> u8 { - (col >> (24u32.strict_sub((row as u32).strict_mul(8)))) as u8 + col.to_be_bytes()[row] } #[cfg(any( @@ -289,6 +291,7 @@ pub(crate) fn aes_enc_round_portable(block: &[u8; BLOCK_SIZE], round_key: &[u8; #[cfg(all(feature = "diag", feature = "aegis256"))] #[must_use] +/// Apply one portable AES encryption round for backend diagnostics. pub fn diag_aes_enc_round_portable(block: &[u8; BLOCK_SIZE], round_key: &[u8; BLOCK_SIZE]) -> [u8; BLOCK_SIZE] { aes_enc_round_portable(block, round_key) } diff --git a/src/aead/ascon128.rs b/src/aead/ascon128.rs index 51cad04a..a6a4cdab 100644 --- a/src/aead/ascon128.rs +++ b/src/aead/ascon128.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! Ascon-AEAD128 authenticated encryption (NIST SP 800-232). //! //! Pure Rust, `no_std` implementation with fixed-work, table-free source @@ -26,17 +24,19 @@ const DOMAIN_SEPARATOR: u64 = 0x8000_0000_0000_0000; /// Little-endian padding: set the first free byte at position `n`. #[inline(always)] -const fn pad(n: usize) -> u64 { - 0x01_u64 << (8 * n) +fn pad(n: usize) -> u64 { + let shift = u32::try_from(n.strict_mul(8)).expect("Ascon tail position must fit the word width"); + 0x01_u64.strict_shl(shift) } /// Clear the lowest `n` bytes of `word`. #[inline(always)] -const fn clear(word: u64, n: usize) -> u64 { +fn clear(word: u64, n: usize) -> u64 { if n == 0 { return word; } - word & (u64::MAX << (8 * n)) + let shift = u32::try_from(n.strict_mul(8)).expect("Ascon tail length must fit the word width"); + word & u64::MAX.strict_shl(shift) } /// Load up to 8 bytes little-endian into a u64, zero-padding on the right. @@ -213,14 +213,13 @@ impl AsconAead128 { /// Absorb associated data into the state. fn process_aad(s: &mut [u64; 5], aad: &[u8]) { if !aad.is_empty() { - let mut chunks = aad.chunks_exact(RATE); - for chunk in chunks.by_ref() { + let (chunks, mut rest) = aad.as_chunks::(); + for chunk in chunks { s[0] ^= load_bytes(&chunk[..8]); s[1] ^= load_bytes(&chunk[8..]); permute_8_portable(s); } - let mut rest = chunks.remainder(); let sidx = if rest.len() >= 8 { s[0] ^= load_bytes(&rest[..8]); rest = &rest[8..]; @@ -289,8 +288,8 @@ impl Aead for AsconAead128 { let mut s = self.initialize(nonce); Self::process_aad(&mut s, aad); - let mut blocks = buffer.chunks_exact_mut(RATE); - for block in blocks.by_ref() { + let (blocks, mut tail) = buffer.as_chunks_mut::(); + for block in blocks { s[0] ^= load_bytes(&block[..8]); block[..8].copy_from_slice(&s[0].to_le_bytes()); s[1] ^= load_bytes(&block[8..]); @@ -298,7 +297,6 @@ impl Aead for AsconAead128 { permute_8_portable(&mut s); } - let mut tail = blocks.into_remainder(); let sidx = if tail.len() >= 8 { s[0] ^= load_bytes(&tail[..8]); tail[..8].copy_from_slice(&s[0].to_le_bytes()); @@ -328,8 +326,8 @@ impl Aead for AsconAead128 { let mut s = self.initialize(nonce); Self::process_aad(&mut s, aad); - let mut blocks = buffer.chunks_exact_mut(RATE); - for block in blocks.by_ref() { + let (blocks, mut tail) = buffer.as_chunks_mut::(); + for block in blocks { let c0 = load_bytes(&block[..8]); block[..8].copy_from_slice(&(s[0] ^ c0).to_le_bytes()); s[0] = c0; @@ -339,7 +337,6 @@ impl Aead for AsconAead128 { permute_8_portable(&mut s); } - let mut tail = blocks.into_remainder(); let sidx = if tail.len() >= 8 { let c0 = load_bytes(&tail[..8]); tail[..8].copy_from_slice(&(s[0] ^ c0).to_le_bytes()); @@ -369,6 +366,7 @@ impl Aead for AsconAead128 { } #[cfg(feature = "diag")] +/// Compare a portable Ascon-AEAD128 tag computation with an expected diagnostic tag. #[unsafe(no_mangle)] #[inline(never)] pub fn diag_ascon_aead128_tag_portable( @@ -402,18 +400,24 @@ mod tests { fn assert_matches_oracle(key: [u8; 16], nonce: [u8; 16], aad: &[u8], plaintext: &[u8]) { let aead = AsconAead128::new(&AsconAead128Key::from_bytes(key)); let nonce_typed = Nonce128::from_bytes(nonce); - let oracle = ascon_aead::AsconAead128::new_from_slice(&key).unwrap(); + let oracle = ascon_aead::AsconAead128::new_from_slice(&key).expect("16-byte Ascon oracle key must be accepted"); let oracle_nonce = Array(nonce); let mut ours = plaintext.to_vec(); - let tag = aead.encrypt_in_place(&nonce_typed, aad, &mut ours).unwrap(); + let tag = aead + .encrypt_in_place(&nonce_typed, aad, &mut ours) + .expect("rscrypto Ascon encryption must succeed"); let mut ours_combined = ours.clone(); ours_combined.extend_from_slice(tag.as_bytes()); - let expected = oracle.encrypt(&oracle_nonce, Payload { msg: plaintext, aad }).unwrap(); + let expected = oracle + .encrypt(&oracle_nonce, Payload { msg: plaintext, aad }) + .expect("oracle Ascon encryption must succeed"); assert_eq!(ours_combined, expected, "encryption mismatch"); let mut ours_buf = ours.clone(); - aead.decrypt_in_place(&nonce_typed, aad, &mut ours_buf, &tag).unwrap(); + aead + .decrypt_in_place(&nonce_typed, aad, &mut ours_buf, &tag) + .expect("rscrypto Ascon self-decryption must succeed"); assert_eq!(ours_buf, plaintext, "self decrypt mismatch"); let (oracle_ct, oracle_tag) = expected.split_at(expected.len().strict_sub(TAG_SIZE)); @@ -423,9 +427,13 @@ mod tests { &nonce_typed, aad, &mut oracle_buf, - &AsconAead128Tag::from_bytes(oracle_tag.try_into().unwrap()), + &AsconAead128Tag::from_bytes( + oracle_tag + .try_into() + .expect("oracle Ascon output must end in a 16-byte tag"), + ), ) - .unwrap(); + .expect("rscrypto must decrypt the oracle Ascon ciphertext"); assert_eq!(oracle_buf, plaintext, "oracle decrypt mismatch"); } @@ -437,8 +445,12 @@ mod tests { let aead = AsconAead128::new(&key); let mut buf = []; - let tag = aead.encrypt_in_place(&nonce, b"", &mut buf).unwrap(); - aead.decrypt_in_place(&nonce, b"", &mut buf, &tag).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"", &mut buf) + .expect("empty Ascon encryption must succeed"); + aead + .decrypt_in_place(&nonce, b"", &mut buf, &tag) + .expect("empty Ascon decryption must succeed"); } #[test] @@ -449,10 +461,14 @@ mod tests { let plaintext = b"the quick brown fox jumps over the lazy dog"; let mut buf = *plaintext; - let tag = aead.encrypt_in_place(&nonce, b"header", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"header", &mut buf) + .expect("Ascon encryption with AAD must succeed"); assert_ne!(&buf[..], &plaintext[..]); - aead.decrypt_in_place(&nonce, b"header", &mut buf, &tag).unwrap(); + aead + .decrypt_in_place(&nonce, b"header", &mut buf, &tag) + .expect("Ascon decryption with AAD must succeed"); assert_eq!(&buf[..], &plaintext[..]); } @@ -465,10 +481,10 @@ mod tests { let mut buf = []; let tag = aead .encrypt_in_place(&nonce, b"associated data only", &mut buf) - .unwrap(); + .expect("AAD-only Ascon encryption must succeed"); aead .decrypt_in_place(&nonce, b"associated data only", &mut buf, &tag) - .unwrap(); + .expect("AAD-only Ascon decryption must succeed"); } #[test] @@ -478,14 +494,18 @@ mod tests { let aead = AsconAead128::new(&key); let mut buf = *b"zero me on failure"; - let tag = aead.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("Ascon test setup encryption must succeed"); let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0xFF; let bad_tag = AsconAead128Tag::from_bytes(bad_tag); - let result = aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert!(buf.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } @@ -496,11 +516,15 @@ mod tests { let aead = AsconAead128::new(&key); let mut buf = *b"secret"; - let tag = aead.encrypt_in_place(&nonce, b"", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"", &mut buf) + .expect("Ascon test setup encryption must succeed"); buf[0] ^= 1; - let result = aead.decrypt_in_place(&nonce, b"", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"", &mut buf, &tag), + Err(OpenError::verification()) + ); // Buffer must be zeroized on failure. assert_eq!(&buf, &[0u8; 6]); } @@ -512,14 +536,18 @@ mod tests { let aead = AsconAead128::new(&key); let mut buf = *b"data"; - let tag = aead.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("Ascon test setup encryption must succeed"); let mut bad_tag_bytes = tag.to_bytes(); bad_tag_bytes[15] ^= 1; let bad_tag = AsconAead128Tag::from_bytes(bad_tag_bytes); - let result = aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert_eq!(&buf, &[0u8; 4]); } @@ -530,10 +558,14 @@ mod tests { let aead = AsconAead128::new(&key); let mut buf = *b"msg"; - let tag = aead.encrypt_in_place(&nonce, b"correct", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"correct", &mut buf) + .expect("Ascon test setup encryption must succeed"); - let result = aead.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag), + Err(OpenError::verification()) + ); } #[test] @@ -543,11 +575,15 @@ mod tests { let aead = AsconAead128::new(&key); let mut buf = *b"nonce test"; - let tag = aead.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("Ascon test setup encryption must succeed"); let wrong_nonce = Nonce128::from_bytes([11; 16]); - let result = aead.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } #[test] @@ -558,18 +594,29 @@ mod tests { let pt = b"combined mode"; let mut sealed = vec![0u8; pt.len().strict_add(TAG_SIZE)]; - aead.encrypt(&nonce, b"h", pt.as_slice(), &mut sealed).unwrap(); + aead + .encrypt(&nonce, b"h", pt.as_slice(), &mut sealed) + .expect("combined Ascon encryption must succeed"); let mut opened = vec![0u8; pt.len()]; - aead.decrypt(&nonce, b"h", &sealed, &mut opened).unwrap(); + aead + .decrypt(&nonce, b"h", &sealed, &mut opened) + .expect("combined Ascon decryption must succeed"); assert_eq!(&opened, &pt[..]); } #[test] fn tag_from_slice_rejects_wrong_length() { - assert!(AsconAead128::tag_from_slice(&[0u8; 15]).is_err()); - assert!(AsconAead128::tag_from_slice(&[0u8; 17]).is_err()); - assert!(AsconAead128::tag_from_slice(&[0u8; 16]).is_ok()); + assert_eq!( + AsconAead128::tag_from_slice(&[0u8; 15]).expect_err("short Ascon tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + AsconAead128::tag_from_slice(&[0u8; 17]).expect_err("long Ascon tag must be rejected"), + AeadBufferError::new() + ); + let tag = AsconAead128::tag_from_slice(&[0u8; 16]).expect("16-byte Ascon tag must be accepted"); + assert_eq!(tag.as_bytes(), &[0u8; 16]); } #[test] @@ -583,7 +630,7 @@ mod tests { let mut buf = plaintext; let tag = aead .encrypt_in_place(&nonce, b"multi-block aad that is longer than one rate block", &mut buf) - .unwrap(); + .expect("multi-block Ascon encryption must succeed"); aead .decrypt_in_place( &nonce, @@ -591,7 +638,7 @@ mod tests { &mut buf, &tag, ) - .unwrap(); + .expect("multi-block Ascon decryption must succeed"); assert_eq!(buf, plaintext); } @@ -604,15 +651,23 @@ mod tests { // Exactly 8 bytes = 1 full block, 0-byte tail. let plaintext = [0x55u8; 8]; let mut buf = plaintext; - let tag = aead.encrypt_in_place(&nonce, b"", &mut buf).unwrap(); - aead.decrypt_in_place(&nonce, b"", &mut buf, &tag).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"", &mut buf) + .expect("one-word Ascon encryption must succeed"); + aead + .decrypt_in_place(&nonce, b"", &mut buf, &tag) + .expect("one-word Ascon decryption must succeed"); assert_eq!(buf, plaintext); // Exactly 16 bytes = 2 full blocks, 0-byte tail. let plaintext16 = [0x66u8; 16]; let mut buf16 = plaintext16; - let tag16 = aead.encrypt_in_place(&nonce, b"", &mut buf16).unwrap(); - aead.decrypt_in_place(&nonce, b"", &mut buf16, &tag16).unwrap(); + let tag16 = aead + .encrypt_in_place(&nonce, b"", &mut buf16) + .expect("one-rate Ascon encryption must succeed"); + aead + .decrypt_in_place(&nonce, b"", &mut buf16, &tag16) + .expect("one-rate Ascon decryption must succeed"); assert_eq!(buf16, plaintext16); } @@ -653,7 +708,7 @@ mod tests { fn differential_multiblock_matches_oracle() { let key = [0x42; 16]; let nonce = [0x24; 16]; - let aad: Vec = (0..48).map(|i| i as u8).collect(); + let aad: Vec = (0u8..48).collect(); let pt: Vec = (0u8..97).map(|i| i.wrapping_mul(17)).collect(); assert_matches_oracle(key, nonce, &aad, &pt); } diff --git a/src/aead/chacha20.rs b/src/aead/chacha20.rs index 091d6374..0a1db21a 100644 --- a/src/aead/chacha20.rs +++ b/src/aead/chacha20.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! Portable ChaCha20 and HChaCha20 core. use core::mem; @@ -25,9 +23,11 @@ pub(crate) const POLY1305_KEY_SIZE: usize = 32; const CONSTANTS: [u32; 4] = [0x6170_7865, 0x3320_646e, 0x7962_2d32, 0x6b20_6574]; -pub(crate) type XorKeystreamFn = fn(&[u8; KEY_SIZE], u32, &[u8; NONCE_SIZE], &mut [u8]); +type XorKeystreamFn = unsafe fn(&[u8; KEY_SIZE], u32, &[u8; NONCE_SIZE], &mut [u8]); +#[cfg(feature = "xchacha20poly1305")] static XCHACHA20POLY1305_XOR_KEYSTREAM_DISPATCH: OnceCache = OnceCache::new(); +#[cfg(feature = "chacha20poly1305")] static CHACHA20POLY1305_XOR_KEYSTREAM_DISPATCH: OnceCache = OnceCache::new(); #[inline(always)] @@ -133,7 +133,9 @@ pub(crate) fn block(key: &[u8; KEY_SIZE], counter: u32, nonce: &[u8; NONCE_SIZE] } let mut out = [0u8; BLOCK_SIZE]; - for (chunk, word) in out.chunks_exact_mut(4).zip(state) { + let (chunks, remainder) = out.as_chunks_mut::<4>(); + debug_assert!(remainder.is_empty()); + for (chunk, word) in chunks.iter_mut().zip(state) { chunk.copy_from_slice(&word.to_le_bytes()); } out @@ -185,19 +187,25 @@ pub(crate) fn xor_keystream( } } - xor_keystream_resolved(primitive)(key, initial_counter, nonce, buffer); + let kernel = xor_keystream_resolved(primitive); + // SAFETY: the counter-range check above covers every 64-byte block, and the resolver returns an accelerated kernel + // only when its complete capability set is available. The portable function also coerces to this unsafe fn type. + unsafe { kernel(key, initial_counter, nonce, buffer) }; Ok(()) } #[inline] -pub(crate) fn xor_keystream_resolved(primitive: AeadPrimitive) -> XorKeystreamFn { +fn xor_keystream_resolved(primitive: AeadPrimitive) -> XorKeystreamFn { match primitive { + #[cfg(feature = "xchacha20poly1305")] AeadPrimitive::XChaCha20Poly1305 => { XCHACHA20POLY1305_XOR_KEYSTREAM_DISPATCH.get_or_init(|| resolve_xor_keystream(primitive)) } + #[cfg(feature = "chacha20poly1305")] AeadPrimitive::ChaCha20Poly1305 => { CHACHA20POLY1305_XOR_KEYSTREAM_DISPATCH.get_or_init(|| resolve_xor_keystream(primitive)) } + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] _ => resolve_xor_keystream(primitive), } } @@ -247,7 +255,7 @@ fn xor_keystream_portable(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u } } -#[cfg_attr(not(all(target_arch = "powerpc64", target_endian = "little")), allow(dead_code))] +#[cfg(all(target_arch = "powerpc64", target_endian = "little"))] pub(crate) fn xor_keystream_first_block_portable( key: &[u8; KEY_SIZE], counter: u32, @@ -265,31 +273,46 @@ pub(crate) fn xor_keystream_first_block_portable( #[cfg(target_arch = "riscv64")] #[inline(always)] -fn simd_u32x4_rotl(value: u32x4, bits: u32) -> u32x4 { - (value << u32x4::splat(bits)) | (value >> u32x4::splat(32u32.wrapping_sub(bits))) +fn simd_u32x4_rotl(value: u32x4) -> u32x4 { + const { assert!(BITS > 0 && BITS < 32) } + let left = core::ops::Shl::shl(value, u32x4::splat(BITS)); + let right = core::ops::Shr::shr(value, u32x4::splat(32u32.strict_sub(BITS))); + core::ops::BitOr::bitor(left, right) +} + +#[cfg(target_arch = "riscv64")] +#[inline(always)] +fn simd_u32x4_wrapping_add(left: u32x4, right: u32x4) -> u32x4 { + core::ops::Add::add(left, right) } #[cfg(target_arch = "riscv64")] #[inline(always)] fn simd_u32x4_quarter_round(a: &mut u32x4, b: &mut u32x4, c: &mut u32x4, d: &mut u32x4) { - *a += *b; + *a = simd_u32x4_wrapping_add(*a, *b); *d ^= *a; - *d = simd_u32x4_rotl(*d, 16); + *d = simd_u32x4_rotl::<16>(*d); - *c += *d; + *c = simd_u32x4_wrapping_add(*c, *d); *b ^= *c; - *b = simd_u32x4_rotl(*b, 12); + *b = simd_u32x4_rotl::<12>(*b); - *a += *b; + *a = simd_u32x4_wrapping_add(*a, *b); *d ^= *a; - *d = simd_u32x4_rotl(*d, 8); + *d = simd_u32x4_rotl::<8>(*d); - *c += *d; + *c = simd_u32x4_wrapping_add(*c, *d); *b ^= *c; - *b = simd_u32x4_rotl(*b, 7); + *b = simd_u32x4_rotl::<7>(*b); } #[cfg(target_arch = "riscv64")] +/// Generate and XOR a ChaCha20 stream in four-block RISC-V vector batches. +/// +/// # Safety +/// +/// The caller must ensure that the RISC-V vector extension is available and that `buffer`'s 64-byte block count fits +/// the counter range starting at `initial_counter`. #[target_feature(enable = "v")] unsafe fn xor_keystream_u32x4_impl( key: &[u8; KEY_SIZE], @@ -298,12 +321,14 @@ unsafe fn xor_keystream_u32x4_impl( buffer: &mut [u8], ) { const BLOCKS_PER_BATCH: usize = 4; + const BLOCKS_PER_BATCH_U32: u32 = 4; + const BATCH_SIZE: usize = BLOCK_SIZE.strict_mul(BLOCKS_PER_BATCH); let mut counter = initial_counter; - let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); - for chunk in &mut batches { + let (batches, remainder) = buffer.as_chunks_mut::(); + for chunk in batches { debug_assert!( - counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some(), + counter <= u32::MAX.strict_sub(BLOCKS_PER_BATCH_U32.strict_sub(1)), "ChaCha20 block counter overflow" ); @@ -321,9 +346,9 @@ unsafe fn xor_keystream_u32x4_impl( let mut x11 = u32x4::splat(load_u32_le(&key[28..32])); let mut x12 = u32x4::from_array([ counter, - counter.wrapping_add(1), - counter.wrapping_add(2), - counter.wrapping_add(3), + counter.strict_add(1), + counter.strict_add(2), + counter.strict_add(3), ]); let mut x13 = u32x4::splat(load_u32_le(&nonce[0..4])); let mut x14 = u32x4::splat(load_u32_le(&nonce[4..8])); @@ -361,22 +386,22 @@ unsafe fn xor_keystream_u32x4_impl( round = round.strict_add(1); } - x0 += o0; - x1 += o1; - x2 += o2; - x3 += o3; - x4 += o4; - x5 += o5; - x6 += o6; - x7 += o7; - x8 += o8; - x9 += o9; - x10 += o10; - x11 += o11; - x12 += o12; - x13 += o13; - x14 += o14; - x15 += o15; + x0 = simd_u32x4_wrapping_add(x0, o0); + x1 = simd_u32x4_wrapping_add(x1, o1); + x2 = simd_u32x4_wrapping_add(x2, o2); + x3 = simd_u32x4_wrapping_add(x3, o3); + x4 = simd_u32x4_wrapping_add(x4, o4); + x5 = simd_u32x4_wrapping_add(x5, o5); + x6 = simd_u32x4_wrapping_add(x6, o6); + x7 = simd_u32x4_wrapping_add(x7, o7); + x8 = simd_u32x4_wrapping_add(x8, o8); + x9 = simd_u32x4_wrapping_add(x9, o9); + x10 = simd_u32x4_wrapping_add(x10, o10); + x11 = simd_u32x4_wrapping_add(x11, o11); + x12 = simd_u32x4_wrapping_add(x12, o12); + x13 = simd_u32x4_wrapping_add(x13, o13); + x14 = simd_u32x4_wrapping_add(x14, o14); + x15 = simd_u32x4_wrapping_add(x15, o15); let words = [ x0.to_array(), @@ -412,10 +437,9 @@ unsafe fn xor_keystream_u32x4_impl( block_index = block_index.strict_add(1); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(BLOCKS_PER_BATCH_U32); } - let remainder = batches.into_remainder(); if !remainder.is_empty() { xor_keystream_portable(key, counter, nonce, remainder); } @@ -429,7 +453,9 @@ pub(crate) fn hchacha20(key: &[u8; KEY_SIZE], nonce: &[u8; HCHACHA_NONCE_SIZE]) rounds(&mut state); let mut out = [0u8; KEY_SIZE]; - for (chunk, word) in out.chunks_exact_mut(4).zip([ + let (chunks, remainder) = out.as_chunks_mut::<4>(); + debug_assert!(remainder.is_empty()); + for (chunk, word) in chunks.iter_mut().zip([ state[0], state[1], state[2], state[3], state[12], state[13], state[14], state[15], ]) { chunk.copy_from_slice(&word.to_le_bytes()); @@ -462,15 +488,22 @@ mod x86_avx512; #[path = "chacha20/x86_64_ssse3_x4.rs"] mod x86_ssse3_x4; -#[cfg(target_arch = "aarch64")] +#[cfg(all(target_arch = "aarch64", feature = "chacha20poly1305"))] #[inline] -pub(crate) fn xor_keystream_aarch64_neon( +/// Apply the AArch64 NEON ChaCha20 kernel for the interleaved AEAD path. +/// +/// # Safety +/// +/// The caller must verify [`aarch64::NEON`](crate::platform::caps::aarch64::NEON) and ensure that the number of +/// 64-byte blocks in `buffer` does not exhaust the `u32` block counter starting at `initial_counter`. +pub(super) unsafe fn xor_keystream_aarch64_neon( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - aarch64_neon::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller establishes the NEON capability and counter-range preconditions required by this backend. + unsafe { aarch64_neon::xor_keystream(key, initial_counter, nonce, buffer) } } // Forced entry points let backend-equivalence tests bypass runtime dispatch. @@ -490,105 +523,120 @@ pub fn diag_chacha20_xor_keystream_portable( /// /// # Safety /// -/// Caller must verify the host has `aarch64::NEON`. Compile-time gated to -/// `target_arch = "aarch64"`. +/// Caller must verify the host has `aarch64::NEON` and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. Compile-time gated to `target_arch = "aarch64"`. #[cfg(all(feature = "diag", target_arch = "aarch64"))] -pub fn diag_chacha20_xor_keystream_aarch64_neon( +pub unsafe fn diag_chacha20_xor_keystream_aarch64_neon( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - aarch64_neon::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { aarch64_neon::xor_keystream(key, initial_counter, nonce, buffer) }; } /// Run the x86_64 AVX2 ChaCha20 XOR-keystream. /// /// # Safety /// -/// Caller must verify the host has `x86::AVX2`. +/// Caller must verify the host has `x86::AVX2` and that `buffer`'s 64-byte block count fits the counter range starting +/// at `initial_counter`. #[cfg(all(feature = "diag", target_arch = "x86_64"))] -pub fn diag_chacha20_xor_keystream_x86_avx2( +pub unsafe fn diag_chacha20_xor_keystream_x86_avx2( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - x86_avx2::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { x86_avx2::xor_keystream(key, initial_counter, nonce, buffer) }; } /// Run the x86_64 AVX-512 ChaCha20 XOR-keystream. /// /// # Safety /// -/// Caller must verify the host has `x86::AVX512F + AVX512VL + AVX512BW`. +/// Caller must verify the host has `x86::AVX512F + AVX512VL + AVX512BW + AVX512DQ` and that `buffer`'s 64-byte block +/// count fits the counter range starting at `initial_counter`. #[cfg(all(feature = "diag", target_arch = "x86_64"))] -pub fn diag_chacha20_xor_keystream_x86_avx512( +pub unsafe fn diag_chacha20_xor_keystream_x86_avx512( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - x86_avx512::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { x86_avx512::xor_keystream(key, initial_counter, nonce, buffer) }; } /// Run the POWER VSX ChaCha20 XOR-keystream. /// /// # Safety /// -/// Caller must verify the host is `powerpc64le` with VSX. The portable -/// kernel — which has been the correctness oracle since commit `2631aefa` -/// fixed the rotation-amount bug here — must produce identical bytes. +/// Caller must verify the host has `power::POWER8_VECTOR` and that `buffer`'s 64-byte block count fits the counter +/// range starting at `initial_counter`. The portable kernel — which has been the correctness oracle since commit +/// `2631aefa` fixed the rotation-amount bug here — must produce identical bytes. #[cfg(all(feature = "diag", target_arch = "powerpc64", target_endian = "little"))] -pub fn diag_chacha20_xor_keystream_power_vsx( +pub unsafe fn diag_chacha20_xor_keystream_power_vsx( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - power_vsx::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { power_vsx::xor_keystream(key, initial_counter, nonce, buffer) }; } /// Run the s390x z/Vector ChaCha20 XOR-keystream. /// /// # Safety /// -/// Caller must verify the host has `s390x::VECTOR`. Same correctness-oracle -/// invariant as POWER VSX above. +/// Caller must verify the host has `s390x::VECTOR` and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. Same correctness-oracle invariant as POWER VSX above. #[cfg(all(feature = "diag", target_arch = "s390x"))] -pub fn diag_chacha20_xor_keystream_s390x_vector( +pub unsafe fn diag_chacha20_xor_keystream_s390x_vector( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - s390x_vector::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { s390x_vector::xor_keystream(key, initial_counter, nonce, buffer) }; } /// Run the riscv64 RVV ChaCha20 XOR-keystream. /// /// # Safety /// -/// Caller must verify the host has `riscv::V`. +/// Caller must verify the host has `riscv::V` and that `buffer`'s 64-byte block count fits the counter range starting +/// at `initial_counter`. #[cfg(all(feature = "diag", target_arch = "riscv64"))] -pub fn diag_chacha20_xor_keystream_riscv64_vector( +pub unsafe fn diag_chacha20_xor_keystream_riscv64_vector( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - riscv64_vector::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { riscv64_vector::xor_keystream(key, initial_counter, nonce, buffer) }; } /// Run the wasm32 simd128 ChaCha20 XOR-keystream. +/// +/// # Safety +/// +/// Caller must verify the host has `wasm::SIMD128` and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. #[cfg(all(feature = "diag", target_arch = "wasm32"))] -pub fn diag_chacha20_xor_keystream_wasm_simd128( +pub unsafe fn diag_chacha20_xor_keystream_wasm_simd128( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - wasm_simd128::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { wasm_simd128::xor_keystream(key, initial_counter, nonce, buffer) }; } #[cfg(test)] mod tests { @@ -611,6 +659,17 @@ mod tests { use super::xor_keystream_portable; use super::{KEY_SIZE, NONCE_SIZE, block, xor_keystream}; use crate::aead::targets::AeadPrimitive; + + fn primitive() -> AeadPrimitive { + #[cfg(feature = "chacha20poly1305")] + { + AeadPrimitive::ChaCha20Poly1305 + } + #[cfg(all(not(feature = "chacha20poly1305"), feature = "xchacha20poly1305"))] + { + AeadPrimitive::XChaCha20Poly1305 + } + } #[cfg(target_arch = "aarch64")] use crate::platform::caps::aarch64; #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] @@ -662,10 +721,12 @@ mod tests { let plaintext = *b"chacha20 portable core"; let mut ciphertext = plaintext; - xor_keystream(AeadPrimitive::ChaCha20Poly1305, &key, 1, &nonce, &mut ciphertext).unwrap(); + xor_keystream(primitive(), &key, 1, &nonce, &mut ciphertext) + .expect("test message must fit the ChaCha20 counter range"); assert_ne!(ciphertext, plaintext); - xor_keystream(AeadPrimitive::XChaCha20Poly1305, &key, 1, &nonce, &mut ciphertext).unwrap(); + xor_keystream(primitive(), &key, 1, &nonce, &mut ciphertext) + .expect("test message must fit the XChaCha20 counter range"); assert_eq!(ciphertext, plaintext); } @@ -675,10 +736,58 @@ mod tests { let nonce = [0u8; NONCE_SIZE]; let mut one_block = [0u8; 64]; - assert!(xor_keystream(AeadPrimitive::ChaCha20Poly1305, &key, u32::MAX, &nonce, &mut one_block).is_ok()); + xor_keystream(primitive(), &key, u32::MAX, &nonce, &mut one_block) + .expect("one block at the final counter must fit"); let mut two_blocks = [0u8; 65]; - assert!(xor_keystream(AeadPrimitive::ChaCha20Poly1305, &key, u32::MAX, &nonce, &mut two_blocks).is_err()); + xor_keystream(primitive(), &key, u32::MAX, &nonce, &mut two_blocks) + .expect_err("a second block after the final counter must fail"); + } + + #[cfg(target_arch = "x86_64")] + /// Compare one capability-gated x86 backend with the portable authority and preserve both guard regions. + /// + /// # Safety + /// + /// The caller must establish `kernel`'s CPU features and ensure that the block count implied by the `len` bytes fits + /// the counter range starting at `initial_counter`. + unsafe fn assert_x86_backend_matches_portable( + backend_name: &str, + kernel: unsafe fn(&[u8; KEY_SIZE], u32, &[u8; NONCE_SIZE], &mut [u8]), + key: &[u8; KEY_SIZE], + nonce: &[u8; NONCE_SIZE], + initial_counter: u32, + offset: usize, + len: usize, + ) { + const GUARD_LEN: usize = 64; + + let allocation_len = GUARD_LEN.strict_add(offset).strict_add(len).strict_add(GUARD_LEN); + let start = GUARD_LEN.strict_add(offset); + let mut portable_storage = vec![0xa5; allocation_len]; + let mut accelerated_storage = portable_storage.clone(); + { + let (_, portable_suffix) = portable_storage.split_at_mut(start); + let (portable, _) = portable_suffix.split_at_mut(len); + let (_, accelerated_suffix) = accelerated_storage.split_at_mut(start); + let (accelerated, _) = accelerated_suffix.split_at_mut(len); + + for (index, (portable_byte, accelerated_byte)) in portable.iter_mut().zip(accelerated.iter_mut()).enumerate() { + let [value, ..] = index.strict_mul(29).strict_add(3).to_le_bytes(); + *portable_byte = value; + *accelerated_byte = value; + } + + xor_keystream_portable(key, initial_counter, nonce, portable); + // SAFETY: the caller established the kernel capability and counter range; `accelerated` is the exact initialized + // in-place segment and the surrounding allocation stays live for the call. + unsafe { kernel(key, initial_counter, nonce, accelerated) }; + } + + assert_eq!( + accelerated_storage, portable_storage, + "{backend_name} mismatch at counter={initial_counter}, offset={offset}, len={len}" + ); } #[test] @@ -690,22 +799,48 @@ mod tests { let key = [0x71; KEY_SIZE]; let nonce = [0x19; NONCE_SIZE]; - for len in [ - 0usize, 1, 63, 64, 65, 255, 256, 257, 511, 512, 513, 768, 769, 1023, 1024, 1280, 1281, 1536, 2048, 4096, 8192, + for initial_counter in [0, 0x7fff_ffff, 0x8000_0000] { + for offset in [0usize, 1, 15, 31, 63] { + for len in [ + 0usize, 1, 63, 64, 65, 255, 256, 257, 511, 512, 513, 768, 769, 1023, 1024, 1279, 1280, 1281, 1536, 2048, + 4096, 8192, + ] { + // SAFETY: the test returned unless AVX512_READY is available; these ordinary/sign-boundary counters leave + // room for every selected test length. + unsafe { + assert_x86_backend_matches_portable( + "AVX-512", + super::x86_avx512::xor_keystream, + &key, + &nonce, + initial_counter, + offset, + len, + ); + } + } + } + } + for (initial_counter, len) in [ + (u32::MAX.strict_sub(15), 1024usize), + (u32::MAX.strict_sub(3), 256), + (u32::MAX, 64), ] { - let mut portable = vec![0u8; len]; - let mut accelerated = vec![0u8; len]; - let mut index = 0usize; - while index < len { - let value = index.strict_mul(13).strict_add(5) as u8; - portable[index] = value; - accelerated[index] = value; - index = index.strict_add(1); + for offset in [0usize, 1, 15, 31, 63] { + // SAFETY: the test returned unless AVX512_READY is available, and each pair is the exact maximum-valid counter + // for the 16-way, four-way, or portable width. + unsafe { + assert_x86_backend_matches_portable( + "AVX-512", + super::x86_avx512::xor_keystream, + &key, + &nonce, + initial_counter, + offset, + len, + ); + } } - - xor_keystream_portable(&key, 3, &nonce, &mut portable); - super::x86_avx512::xor_keystream(&key, 3, &nonce, &mut accelerated); - assert_eq!(accelerated, portable, "AVX-512 mismatch at len={len}"); } } @@ -718,22 +853,48 @@ mod tests { let key = [0x55; KEY_SIZE]; let nonce = [0x33; NONCE_SIZE]; - for len in [ - 0usize, 1, 63, 64, 65, 255, 256, 257, 511, 512, 513, 768, 769, 1023, 1024, 1280, 1281, 2048, 4096, 8192, + for initial_counter in [0, 0x7fff_ffff, 0x8000_0000] { + for offset in [0usize, 1, 15, 31, 63] { + for len in [ + 0usize, 1, 63, 64, 65, 255, 256, 257, 511, 512, 513, 767, 768, 769, 1023, 1024, 1279, 1280, 1281, 2048, 4096, + 8192, + ] { + // SAFETY: the test returned unless AVX2 is available; these ordinary/sign-boundary counters leave room for + // every selected test length. + unsafe { + assert_x86_backend_matches_portable( + "AVX2", + super::x86_avx2::xor_keystream, + &key, + &nonce, + initial_counter, + offset, + len, + ); + } + } + } + } + for (initial_counter, len) in [ + (u32::MAX.strict_sub(7), 512usize), + (u32::MAX.strict_sub(3), 256), + (u32::MAX, 64), ] { - let mut portable = vec![0u8; len]; - let mut accelerated = vec![0u8; len]; - let mut index = 0usize; - while index < len { - let value = index.strict_mul(17).strict_add(9) as u8; - portable[index] = value; - accelerated[index] = value; - index = index.strict_add(1); + for offset in [0usize, 1, 15, 31, 63] { + // SAFETY: the test returned unless AVX2 is available, and each pair is the exact maximum-valid counter for + // the eight-way, four-way, or portable width. + unsafe { + assert_x86_backend_matches_portable( + "AVX2", + super::x86_avx2::xor_keystream, + &key, + &nonce, + initial_counter, + offset, + len, + ); + } } - - xor_keystream_portable(&key, 7, &nonce, &mut portable); - super::x86_avx2::xor_keystream(&key, 7, &nonce, &mut accelerated); - assert_eq!(accelerated, portable, "AVX2 mismatch at len={len}"); } } @@ -746,20 +907,32 @@ mod tests { let key = [0x66; KEY_SIZE]; let nonce = [0x11; NONCE_SIZE]; - for len in [0usize, 1, 63, 64, 65, 127, 128, 129, 255, 256, 257, 768] { - let mut portable = vec![0u8; len]; - let mut accelerated = vec![0u8; len]; - let mut index = 0usize; - while index < len { - let value = index.strict_mul(29).strict_add(3) as u8; - portable[index] = value; - accelerated[index] = value; - index = index.strict_add(1); + for initial_counter in [0, 11, u32::MAX.strict_sub(11)] { + for offset in 0usize..16 { + for len in [0usize, 1, 63, 64, 65, 127, 128, 129, 255, 256, 257, 511, 512, 513, 768] { + let allocation_len = offset.strict_add(len); + let mut portable_storage = vec![0u8; allocation_len]; + let mut accelerated_storage = vec![0u8; allocation_len]; + let (_, portable) = portable_storage.split_at_mut(offset); + let (_, accelerated) = accelerated_storage.split_at_mut(offset); + let mut index = 0usize; + while index < len { + let [value, ..] = index.strict_mul(29).strict_add(3).to_le_bytes(); + portable[index] = value; + accelerated[index] = value; + index = index.strict_add(1); + } + + xor_keystream_portable(&key, initial_counter, &nonce, portable); + // SAFETY: the test capability guard proves NEON, and the selected maximum counter leaves room for every + // selected length. + unsafe { super::aarch64_neon::xor_keystream(&key, initial_counter, &nonce, accelerated) }; + assert_eq!( + accelerated, portable, + "NEON mismatch at counter={initial_counter}, offset={offset}, len={len}" + ); + } } - - xor_keystream_portable(&key, 11, &nonce, &mut portable); - super::aarch64_neon::xor_keystream(&key, 11, &nonce, &mut accelerated); - assert_eq!(accelerated, portable); } } @@ -784,7 +957,8 @@ mod tests { } xor_keystream_portable(&key, 5, &nonce, &mut portable); - super::power_vsx::xor_keystream(&key, 5, &nonce, &mut accelerated); + // SAFETY: the test capability guard proves POWER8 vector support, and counter 5 leaves room for every length. + unsafe { super::power_vsx::xor_keystream(&key, 5, &nonce, &mut accelerated) }; assert_eq!(accelerated, portable, "POWER VSX mismatch at len={len}"); } } @@ -810,7 +984,8 @@ mod tests { } xor_keystream_portable(&key, 9, &nonce, &mut portable); - super::s390x_vector::xor_keystream(&key, 9, &nonce, &mut accelerated); + // SAFETY: the test capability guard proves z/Vector support, and counter 9 leaves room for every length. + unsafe { super::s390x_vector::xor_keystream(&key, 9, &nonce, &mut accelerated) }; assert_eq!(accelerated, portable, "s390x vector mismatch at len={len}"); } } diff --git a/src/aead/chacha20/aarch64_neon.rs b/src/aead/chacha20/aarch64_neon.rs index de857391..4b105f72 100644 --- a/src/aead/chacha20/aarch64_neon.rs +++ b/src/aead/chacha20/aarch64_neon.rs @@ -1,21 +1,37 @@ use core::arch::aarch64::{ - uint32x4_t, vaddq_u32, vcombine_u32, vdupq_n_u32, veorq_u32, vget_high_u32, vget_low_u32, vld1q_u32, - vreinterpretq_u16_u32, vreinterpretq_u32_u16, vrev32q_u16, vshrq_n_u32, vsliq_n_u32, vst1q_u32, vzip1q_u32, - vzip2q_u32, + uint32x4_t, vaddq_u32, vcombine_u32, vdupq_n_u32, veorq_u32, vget_high_u32, vget_low_u32, vld1q_u8, vld1q_u32, + vreinterpretq_u8_u32, vreinterpretq_u16_u32, vreinterpretq_u32_u8, vreinterpretq_u32_u16, vrev32q_u16, vshrq_n_u32, + vsliq_n_u32, vst1q_u8, vzip1q_u32, vzip2q_u32, }; use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le, xor_keystream_portable}; const BLOCKS_PER_BATCH: usize = 4; - +const COUNTERS_PER_BATCH: u32 = 4; +const COUNTERS_PER_DOUBLE_BATCH: u32 = 8; + +/// Generate and XOR a ChaCha20 stream with the NEON kernel. +/// +/// # Safety +/// +/// The caller must ensure that NEON is available and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. #[inline] -pub(super) fn xor_keystream(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { - // SAFETY: Backend selection guarantees NEON is available before this wrapper is chosen because: - // 1. Runtime dispatch selects this module only after `aarch64::NEON` is present. - // 2. `xor_keystream_impl` is annotated with `#[target_feature(enable = "neon")]`. +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: Production validates the counter range and detects `aarch64::NEON`; direct test and diagnostic callers + // establish the same conditions. `xor_keystream_impl` enables NEON. unsafe { xor_keystream_impl(key, initial_counter, nonce, buffer) } } +/// # Safety +/// +/// The caller must ensure that NEON is available and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. #[target_feature(enable = "neon")] unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { let c0 = vdupq_n_u32(0x6170_7865); @@ -37,7 +53,7 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: let mut counter = initial_counter; let mut double_batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH * 2); for chunk in &mut double_batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH * 2 - 1) as u32).is_some()); + debug_assert!(counter.checked_add(COUNTERS_PER_DOUBLE_BATCH.strict_sub(1)).is_some()); let mut x0 = c0; let mut x1 = c1; @@ -175,14 +191,14 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: xor_store_word_group(hi, 12, y12, y13, y14, y15); } - counter = counter.wrapping_add((BLOCKS_PER_BATCH * 2) as u32); + counter = counter.wrapping_add(COUNTERS_PER_DOUBLE_BATCH); } let mut batches = double_batches .into_remainder() .chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); for chunk in &mut batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some()); + debug_assert!(counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); let mut x0 = c0; let mut x1 = c1; @@ -279,7 +295,7 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: xor_store_word_group(ptr, 12, x12, x13, x14, x15); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(COUNTERS_PER_BATCH); } let remainder = batches.into_remainder(); @@ -289,6 +305,11 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: } /// Transpose four word-major ChaCha vectors into four block-major 16-byte word groups. +/// +/// # Safety +/// +/// The caller must ensure NEON is available, `chunk` is valid for exclusive writes across four complete ChaCha20 +/// blocks, and `word_start` is one of 0, 4, 8, or 12. #[inline(always)] unsafe fn xor_store_word_group( chunk: *mut u8, @@ -320,18 +341,25 @@ unsafe fn xor_store_word_group( } } +/// XOR one 16-byte word group into a block of a four-block ChaCha20 batch. +/// +/// # Safety +/// +/// The caller must ensure NEON is available, `chunk` is valid for exclusive writes across four complete ChaCha20 +/// blocks, `block_index < 4`, and `word_start <= 12`. #[inline(always)] unsafe fn xor_store_block_words(chunk: *mut u8, block_index: usize, word_start: usize, keystream: uint32x4_t) { // SAFETY: in-place 16-byte XOR/store because: // 1. Caller guarantees `chunk` points to a full four-block chunk. // 2. `block_index < 4` and `word_start <= 12`, so `block_index * 64 + word_start * 4 + 16` stays // within the 256-byte chunk. - // 3. `vld1q_u32`/`vst1q_u32` support unaligned addresses and the pointer does not escape. + // 3. The byte-typed NEON load/store accepts every address alignment and the pointer does not escape. unsafe { let offset = block_index.strict_mul(BLOCK_SIZE).strict_add(word_start.strict_mul(4)); - let ptr = chunk.add(offset).cast::(); - let plaintext = vld1q_u32(ptr); - vst1q_u32(ptr, veorq_u32(plaintext, keystream)); + let ptr = chunk.add(offset); + let plaintext = vreinterpretq_u32_u8(vld1q_u8(ptr)); + let ciphertext = veorq_u32(plaintext, keystream); + vst1q_u8(ptr, vreinterpretq_u8_u32(ciphertext)); } } diff --git a/src/aead/chacha20/powerpc64_vsx.rs b/src/aead/chacha20/powerpc64_vsx.rs index 151c2b95..52544af7 100644 --- a/src/aead/chacha20/powerpc64_vsx.rs +++ b/src/aead/chacha20/powerpc64_vsx.rs @@ -3,10 +3,23 @@ use core::simd::i64x2; use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le, xor_keystream_portable}; const BLOCKS_PER_BATCH: usize = 4; +const COUNTERS_PER_BATCH: u32 = 4; +/// Generate and XOR a ChaCha20 stream with the POWER8 vector kernel. +/// +/// # Safety +/// +/// The caller must ensure that POWER8 vector support is available and that `buffer`'s 64-byte block count fits the +/// counter range starting at `initial_counter`. #[inline] -pub(super) fn xor_keystream(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { - // SAFETY: Backend selection guarantees POWER vector support before this wrapper is chosen. +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: Production validates the counter range and detects `power::POWER8_VECTOR`; direct test and diagnostic + // callers establish the same conditions. unsafe { xor_keystream_impl(key, initial_counter, nonce, buffer) } } @@ -42,14 +55,20 @@ impl RotShifts { } } +/// Generate and XOR a ChaCha20 stream in four-block POWER8 vector batches. +/// +/// # Safety +/// +/// The caller must ensure that POWER8 vector support is available and that `buffer`'s 64-byte block count fits the +/// counter range starting at `initial_counter`. #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { let rot = RotShifts::new(); let mut counter = initial_counter; - let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); - for chunk in &mut batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some()); + let (batches, remainder) = buffer.as_chunks_mut::<{ BLOCK_SIZE * BLOCKS_PER_BATCH }>(); + for chunk in batches { + debug_assert!(counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); let mut x0 = splat(0x6170_7865); let mut x1 = splat(0x3320_646e); @@ -162,10 +181,9 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: block_index = block_index.strict_add(1); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(COUNTERS_PER_BATCH); } - let remainder = batches.into_remainder(); if !remainder.is_empty() { xor_keystream_portable(key, counter, nonce, remainder); } @@ -189,6 +207,10 @@ fn splat(val: u32) -> i64x2 { } /// Vector add unsigned word modulo: `vadduwm`. +/// +/// # Safety +/// +/// The executing CPU must support POWER8 vector instructions. #[inline(always)] unsafe fn vadduwm(a: i64x2, b: i64x2) -> i64x2 { let out: i64x2; @@ -206,6 +228,10 @@ unsafe fn vadduwm(a: i64x2, b: i64x2) -> i64x2 { } /// Vector XOR: `vxor`. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec. #[inline(always)] unsafe fn vxor(a: i64x2, b: i64x2) -> i64x2 { let out: i64x2; @@ -223,6 +249,10 @@ unsafe fn vxor(a: i64x2, b: i64x2) -> i64x2 { } /// Vector rotate left word: `vrlw`. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec. #[inline(always)] unsafe fn vrlw(value: i64x2, shift: i64x2) -> i64x2 { let out: i64x2; @@ -239,6 +269,11 @@ unsafe fn vrlw(value: i64x2, shift: i64x2) -> i64x2 { out } +/// Apply one vectorized ChaCha20 quarter round. +/// +/// # Safety +/// +/// The executing CPU must support POWER8 vector instructions. #[inline(always)] unsafe fn quarter_round(a: &mut i64x2, b: &mut i64x2, c: &mut i64x2, d: &mut i64x2, rot: &RotShifts) { // SAFETY: POWER8+ VSX available via enclosing target_feature. diff --git a/src/aead/chacha20/riscv64_vector.rs b/src/aead/chacha20/riscv64_vector.rs index 77516044..2a7376d6 100644 --- a/src/aead/chacha20/riscv64_vector.rs +++ b/src/aead/chacha20/riscv64_vector.rs @@ -1,15 +1,18 @@ use super::{KEY_SIZE, NONCE_SIZE, xor_keystream_u32x4_impl}; -define_target_feature_forwarder! { - pub(super) fn xor_keystream( - key: &[u8; KEY_SIZE], - initial_counter: u32, - nonce: &[u8; NONCE_SIZE], - buffer: &mut [u8] - ) { - feature = "v"; - outer_safety = "backend selection guarantees the vector extension before this wrapper is chosen."; - inner_safety = "the wrapper only reaches this function when the RISC-V vector extension is available."; - call = xor_keystream_u32x4_impl(key, initial_counter, nonce, buffer); - } +/// Generate and XOR a ChaCha20 stream with the RISC-V vector kernel. +/// +/// # Safety +/// +/// The caller must ensure that the RISC-V vector extension is available and that `buffer`'s 64-byte block count fits +/// the counter range starting at `initial_counter`. +#[inline] +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: the caller established the vector capability and counter range required by the shared implementation. + unsafe { xor_keystream_u32x4_impl(key, initial_counter, nonce, buffer) } } diff --git a/src/aead/chacha20/s390x_vector.rs b/src/aead/chacha20/s390x_vector.rs index 1f82eab7..5cb78603 100644 --- a/src/aead/chacha20/s390x_vector.rs +++ b/src/aead/chacha20/s390x_vector.rs @@ -3,19 +3,38 @@ use core::simd::i64x2; use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le, xor_keystream_portable}; const BLOCKS_PER_BATCH: usize = 4; +const COUNTERS_PER_BATCH: u32 = 4; +/// Generate and XOR a ChaCha20 stream with the z/Vector kernel. +/// +/// # Safety +/// +/// The caller must ensure that the z/Vector facility is available and that `buffer`'s 64-byte block count fits the +/// counter range starting at `initial_counter`. #[inline] -pub(super) fn xor_keystream(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { - // SAFETY: Backend selection guarantees the z/Vector facility before this wrapper is chosen. +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: Production validates the counter range and detects `s390x::VECTOR`; direct test and diagnostic callers + // establish the same conditions. unsafe { xor_keystream_impl(key, initial_counter, nonce, buffer) } } +/// Generate and XOR a ChaCha20 stream in four-block z/Vector batches. +/// +/// # Safety +/// +/// The caller must ensure that the z/Vector facility is available and that `buffer`'s 64-byte block count fits the +/// counter range starting at `initial_counter`. #[target_feature(enable = "vector")] unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { let mut counter = initial_counter; - let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); - for chunk in &mut batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some()); + let (batches, remainder) = buffer.as_chunks_mut::<{ BLOCK_SIZE * BLOCKS_PER_BATCH }>(); + for chunk in batches { + debug_assert!(counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); let mut x0 = splat(0x6170_7865); let mut x1 = splat(0x3320_646e); @@ -128,10 +147,9 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: block_index = block_index.strict_add(1); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(COUNTERS_PER_BATCH); } - let remainder = batches.into_remainder(); if !remainder.is_empty() { xor_keystream_portable(key, counter, nonce, remainder); } @@ -155,6 +173,10 @@ fn splat(val: u32) -> i64x2 { } /// Vector add fullword: `vaf`. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn vaf(a: i64x2, b: i64x2) -> i64x2 { @@ -173,6 +195,10 @@ unsafe fn vaf(a: i64x2, b: i64x2) -> i64x2 { } /// Vector exclusive OR: `vx`. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn vx(a: i64x2, b: i64x2) -> i64x2 { @@ -194,6 +220,10 @@ unsafe fn vx(a: i64x2, b: i64x2) -> i64x2 { /// /// ChaCha20 quarter rounds rotate LEFT by 16, 12, 8, and 7 bits, and `verll` /// already rotates LEFT. The immediate maps directly to the ChaCha constant. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn rotl32_via_verll(a: i64x2) -> i64x2 { @@ -215,6 +245,11 @@ unsafe fn rotl32_via_verll(a: i64x2) -> i64x2 { out } +/// Apply one vectorized ChaCha20 quarter round. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn quarter_round(a: &mut i64x2, b: &mut i64x2, c: &mut i64x2, d: &mut i64x2) { diff --git a/src/aead/chacha20/wasm32_simd128.rs b/src/aead/chacha20/wasm32_simd128.rs index 128e6290..aa853f1b 100644 --- a/src/aead/chacha20/wasm32_simd128.rs +++ b/src/aead/chacha20/wasm32_simd128.rs @@ -4,18 +4,36 @@ use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le, xor_keystream_portabl const BLOCKS_PER_BATCH: usize = 4; +/// Generate and XOR a ChaCha20 stream with the wasm SIMD128 kernel. +/// +/// # Safety +/// +/// The caller must ensure that SIMD128 is available and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. #[inline] -pub(super) fn xor_keystream(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { - // SAFETY: Backend selection guarantees simd128 is available before this wrapper is chosen. +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: Production validates the counter range and selects this wrapper only when simd128 is available; direct + // test and diagnostic callers establish the same conditions. unsafe { xor_keystream_impl(key, initial_counter, nonce, buffer) } } +/// Generate and XOR a ChaCha20 stream in four-block SIMD128 batches. +/// +/// # Safety +/// +/// The caller must ensure that SIMD128 is available and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. #[target_feature(enable = "simd128")] unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { let mut counter = initial_counter; let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); for chunk in &mut batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some()); + debug_assert!(counter.checked_add(3).is_some()); let mut x0 = u32x4_splat(0x6170_7865); let mut x1 = u32x4_splat(0x3320_646e); @@ -88,26 +106,24 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: x14 = u32x4_add(x14, o14); x15 = u32x4_add(x15, o15); - let mut words = [[0u32; BLOCKS_PER_BATCH]; 16]; - // SAFETY: each destination is a valid four-lane `u32` array for one unaligned `v128` store. - unsafe { - v128_store(words[0].as_mut_ptr() as *mut v128, x0); - v128_store(words[1].as_mut_ptr() as *mut v128, x1); - v128_store(words[2].as_mut_ptr() as *mut v128, x2); - v128_store(words[3].as_mut_ptr() as *mut v128, x3); - v128_store(words[4].as_mut_ptr() as *mut v128, x4); - v128_store(words[5].as_mut_ptr() as *mut v128, x5); - v128_store(words[6].as_mut_ptr() as *mut v128, x6); - v128_store(words[7].as_mut_ptr() as *mut v128, x7); - v128_store(words[8].as_mut_ptr() as *mut v128, x8); - v128_store(words[9].as_mut_ptr() as *mut v128, x9); - v128_store(words[10].as_mut_ptr() as *mut v128, x10); - v128_store(words[11].as_mut_ptr() as *mut v128, x11); - v128_store(words[12].as_mut_ptr() as *mut v128, x12); - v128_store(words[13].as_mut_ptr() as *mut v128, x13); - v128_store(words[14].as_mut_ptr() as *mut v128, x14); - v128_store(words[15].as_mut_ptr() as *mut v128, x15); - } + let words = [ + lanes(x0), + lanes(x1), + lanes(x2), + lanes(x3), + lanes(x4), + lanes(x5), + lanes(x6), + lanes(x7), + lanes(x8), + lanes(x9), + lanes(x10), + lanes(x11), + lanes(x12), + lanes(x13), + lanes(x14), + lanes(x15), + ]; let mut block_index = 0usize; while block_index < BLOCKS_PER_BATCH { @@ -124,7 +140,7 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: block_index = block_index.strict_add(1); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(4); } let remainder = batches.into_remainder(); @@ -133,6 +149,16 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: } } +#[inline(always)] +fn lanes(value: v128) -> [u32; BLOCKS_PER_BATCH] { + [ + u32x4_extract_lane::<0>(value), + u32x4_extract_lane::<1>(value), + u32x4_extract_lane::<2>(value), + u32x4_extract_lane::<3>(value), + ] +} + #[inline(always)] fn quarter_round(a: &mut v128, b: &mut v128, c: &mut v128, d: &mut v128) { *a = u32x4_add(*a, *b); diff --git a/src/aead/chacha20/x86_64_avx2.rs b/src/aead/chacha20/x86_64_avx2.rs index 5165bce0..6ee8a08b 100644 --- a/src/aead/chacha20/x86_64_avx2.rs +++ b/src/aead/chacha20/x86_64_avx2.rs @@ -7,13 +7,32 @@ use core::arch::x86_64::{ use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le, x86_ssse3_x4, xor_keystream_portable}; const BLOCKS_PER_BATCH: usize = 8; +const COUNTERS_PER_BATCH: u32 = 8; +/// Generate and XOR a ChaCha20 stream with the AVX2 kernel. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. #[inline] -pub(super) fn xor_keystream(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { - // SAFETY: Backend selection guarantees AVX2 is available before this wrapper is chosen. +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: Production validates the counter range and detects AVX2; direct test and diagnostic callers establish the + // same conditions. unsafe { xor_keystream_impl(key, initial_counter, nonce, buffer) } } +/// Generate and XOR a ChaCha20 stream with the AVX2 eight-way kernel and four-way/portable tails. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available and that the number of 64-byte blocks in `buffer` does not exhaust +/// the 32-bit block counter starting at `initial_counter`. #[target_feature(enable = "avx2")] unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { // vpshufb masks for byte-aligned rotations (16-bit and 8-bit). @@ -27,33 +46,33 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: let mut counter = initial_counter; let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); for chunk in &mut batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some()); - - let mut x0 = _mm256_set1_epi32(0x6170_7865u32 as i32); - let mut x1 = _mm256_set1_epi32(0x3320_646eu32 as i32); - let mut x2 = _mm256_set1_epi32(0x7962_2d32u32 as i32); - let mut x3 = _mm256_set1_epi32(0x6b20_6574u32 as i32); - let mut x4 = _mm256_set1_epi32(load_u32_le(&key[0..4]) as i32); - let mut x5 = _mm256_set1_epi32(load_u32_le(&key[4..8]) as i32); - let mut x6 = _mm256_set1_epi32(load_u32_le(&key[8..12]) as i32); - let mut x7 = _mm256_set1_epi32(load_u32_le(&key[12..16]) as i32); - let mut x8 = _mm256_set1_epi32(load_u32_le(&key[16..20]) as i32); - let mut x9 = _mm256_set1_epi32(load_u32_le(&key[20..24]) as i32); - let mut x10 = _mm256_set1_epi32(load_u32_le(&key[24..28]) as i32); - let mut x11 = _mm256_set1_epi32(load_u32_le(&key[28..32]) as i32); + debug_assert!(counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); + + let mut x0 = _mm256_set1_epi32(0x6170_7865u32.cast_signed()); + let mut x1 = _mm256_set1_epi32(0x3320_646eu32.cast_signed()); + let mut x2 = _mm256_set1_epi32(0x7962_2d32u32.cast_signed()); + let mut x3 = _mm256_set1_epi32(0x6b20_6574u32.cast_signed()); + let mut x4 = _mm256_set1_epi32(load_u32_le(&key[0..4]).cast_signed()); + let mut x5 = _mm256_set1_epi32(load_u32_le(&key[4..8]).cast_signed()); + let mut x6 = _mm256_set1_epi32(load_u32_le(&key[8..12]).cast_signed()); + let mut x7 = _mm256_set1_epi32(load_u32_le(&key[12..16]).cast_signed()); + let mut x8 = _mm256_set1_epi32(load_u32_le(&key[16..20]).cast_signed()); + let mut x9 = _mm256_set1_epi32(load_u32_le(&key[20..24]).cast_signed()); + let mut x10 = _mm256_set1_epi32(load_u32_le(&key[24..28]).cast_signed()); + let mut x11 = _mm256_set1_epi32(load_u32_le(&key[28..32]).cast_signed()); let mut x12 = _mm256_setr_epi32( - counter as i32, - counter.wrapping_add(1) as i32, - counter.wrapping_add(2) as i32, - counter.wrapping_add(3) as i32, - counter.wrapping_add(4) as i32, - counter.wrapping_add(5) as i32, - counter.wrapping_add(6) as i32, - counter.wrapping_add(7) as i32, + counter.cast_signed(), + counter.wrapping_add(1).cast_signed(), + counter.wrapping_add(2).cast_signed(), + counter.wrapping_add(3).cast_signed(), + counter.wrapping_add(4).cast_signed(), + counter.wrapping_add(5).cast_signed(), + counter.wrapping_add(6).cast_signed(), + counter.wrapping_add(7).cast_signed(), ); - let mut x13 = _mm256_set1_epi32(load_u32_le(&nonce[0..4]) as i32); - let mut x14 = _mm256_set1_epi32(load_u32_le(&nonce[4..8]) as i32); - let mut x15 = _mm256_set1_epi32(load_u32_le(&nonce[8..12]) as i32); + let mut x13 = _mm256_set1_epi32(load_u32_le(&nonce[0..4]).cast_signed()); + let mut x14 = _mm256_set1_epi32(load_u32_le(&nonce[4..8]).cast_signed()); + let mut x15 = _mm256_set1_epi32(load_u32_le(&nonce[8..12]).cast_signed()); let o0 = x0; let o1 = x1; @@ -108,7 +127,6 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: // to block-major (each pair of YMM registers = one 64-byte block). // // Stage 1: 32-bit interleave. - // SAFETY: AVX2 intrinsics are valid under the enclosing target_feature. let s1_0 = _mm256_unpacklo_epi32(x0, x1); let s1_1 = _mm256_unpackhi_epi32(x0, x1); let s1_2 = _mm256_unpacklo_epi32(x2, x3); @@ -162,7 +180,7 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: xor_block_pair(ptr, 3, 7, s2_3, s2_7, s2_11, s2_15); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(COUNTERS_PER_BATCH); } let remainder = batches.into_remainder(); @@ -171,7 +189,7 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: // SAFETY: AVX2-capable CPUs provide the SSSE3 instructions used by the // 4-block tail kernel, and `chunk` is exactly 4 ChaCha20 blocks. unsafe { x86_ssse3_x4::xor_blocks(key, counter, nonce, chunk) }; - counter = counter.wrapping_add(x86_ssse3_x4::BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(x86_ssse3_x4::COUNTERS_PER_BATCH); } let remainder = x4_batches.into_remainder(); @@ -182,6 +200,11 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: /// Permute stage-2 results into two complete blocks (lo_idx and hi_idx) and /// XOR+store them in-place. Each block is 64 bytes = 2 × YMM. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available, `buf` is valid for exclusive access to 512 initialized bytes, +/// and `lo_idx` and `hi_idx` are distinct values below eight. #[inline(always)] unsafe fn xor_block_pair( buf: *mut u8, diff --git a/src/aead/chacha20/x86_64_avx512.rs b/src/aead/chacha20/x86_64_avx512.rs index 8f8c49a7..6635f62f 100644 --- a/src/aead/chacha20/x86_64_avx512.rs +++ b/src/aead/chacha20/x86_64_avx512.rs @@ -7,53 +7,72 @@ use core::arch::x86_64::{ use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le, x86_ssse3_x4, xor_keystream_portable}; const BLOCKS_PER_BATCH: usize = 16; +const COUNTERS_PER_BATCH: u32 = 16; +/// Generate and XOR a ChaCha20 stream with the AVX-512 kernel. +/// +/// # Safety +/// +/// The caller must ensure that AVX512F, AVX512VL, AVX512BW, and AVX512DQ are available and that `buffer`'s 64-byte +/// block count fits the counter range starting at `initial_counter`. #[inline] -pub(super) fn xor_keystream(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { - // SAFETY: Backend selection guarantees the AVX-512 feature set required by this kernel. +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: Production validates the counter range and detects AVX512F+VL+BW+DQ; direct test and diagnostic callers + // establish the same conditions. unsafe { xor_keystream_impl(key, initial_counter, nonce, buffer) } } +/// Generate and XOR a ChaCha20 stream with the AVX-512 16-way kernel and four-way/portable tails. +/// +/// # Safety +/// +/// The caller must ensure that AVX512F, AVX512VL, AVX512BW, and AVX512DQ are available and that the number of +/// 64-byte blocks in `buffer` does not exhaust the 32-bit block counter starting at `initial_counter`. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq")] unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { let mut counter = initial_counter; let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); for chunk in &mut batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some()); + debug_assert!(counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); - let mut x0 = _mm512_set1_epi32(0x6170_7865u32 as i32); - let mut x1 = _mm512_set1_epi32(0x3320_646eu32 as i32); - let mut x2 = _mm512_set1_epi32(0x7962_2d32u32 as i32); - let mut x3 = _mm512_set1_epi32(0x6b20_6574u32 as i32); - let mut x4 = _mm512_set1_epi32(load_u32_le(&key[0..4]) as i32); - let mut x5 = _mm512_set1_epi32(load_u32_le(&key[4..8]) as i32); - let mut x6 = _mm512_set1_epi32(load_u32_le(&key[8..12]) as i32); - let mut x7 = _mm512_set1_epi32(load_u32_le(&key[12..16]) as i32); - let mut x8 = _mm512_set1_epi32(load_u32_le(&key[16..20]) as i32); - let mut x9 = _mm512_set1_epi32(load_u32_le(&key[20..24]) as i32); - let mut x10 = _mm512_set1_epi32(load_u32_le(&key[24..28]) as i32); - let mut x11 = _mm512_set1_epi32(load_u32_le(&key[28..32]) as i32); + let mut x0 = _mm512_set1_epi32(0x6170_7865u32.cast_signed()); + let mut x1 = _mm512_set1_epi32(0x3320_646eu32.cast_signed()); + let mut x2 = _mm512_set1_epi32(0x7962_2d32u32.cast_signed()); + let mut x3 = _mm512_set1_epi32(0x6b20_6574u32.cast_signed()); + let mut x4 = _mm512_set1_epi32(load_u32_le(&key[0..4]).cast_signed()); + let mut x5 = _mm512_set1_epi32(load_u32_le(&key[4..8]).cast_signed()); + let mut x6 = _mm512_set1_epi32(load_u32_le(&key[8..12]).cast_signed()); + let mut x7 = _mm512_set1_epi32(load_u32_le(&key[12..16]).cast_signed()); + let mut x8 = _mm512_set1_epi32(load_u32_le(&key[16..20]).cast_signed()); + let mut x9 = _mm512_set1_epi32(load_u32_le(&key[20..24]).cast_signed()); + let mut x10 = _mm512_set1_epi32(load_u32_le(&key[24..28]).cast_signed()); + let mut x11 = _mm512_set1_epi32(load_u32_le(&key[28..32]).cast_signed()); let mut x12 = _mm512_setr_epi32( - counter as i32, - counter.wrapping_add(1) as i32, - counter.wrapping_add(2) as i32, - counter.wrapping_add(3) as i32, - counter.wrapping_add(4) as i32, - counter.wrapping_add(5) as i32, - counter.wrapping_add(6) as i32, - counter.wrapping_add(7) as i32, - counter.wrapping_add(8) as i32, - counter.wrapping_add(9) as i32, - counter.wrapping_add(10) as i32, - counter.wrapping_add(11) as i32, - counter.wrapping_add(12) as i32, - counter.wrapping_add(13) as i32, - counter.wrapping_add(14) as i32, - counter.wrapping_add(15) as i32, + counter.cast_signed(), + counter.wrapping_add(1).cast_signed(), + counter.wrapping_add(2).cast_signed(), + counter.wrapping_add(3).cast_signed(), + counter.wrapping_add(4).cast_signed(), + counter.wrapping_add(5).cast_signed(), + counter.wrapping_add(6).cast_signed(), + counter.wrapping_add(7).cast_signed(), + counter.wrapping_add(8).cast_signed(), + counter.wrapping_add(9).cast_signed(), + counter.wrapping_add(10).cast_signed(), + counter.wrapping_add(11).cast_signed(), + counter.wrapping_add(12).cast_signed(), + counter.wrapping_add(13).cast_signed(), + counter.wrapping_add(14).cast_signed(), + counter.wrapping_add(15).cast_signed(), ); - let mut x13 = _mm512_set1_epi32(load_u32_le(&nonce[0..4]) as i32); - let mut x14 = _mm512_set1_epi32(load_u32_le(&nonce[4..8]) as i32); - let mut x15 = _mm512_set1_epi32(load_u32_le(&nonce[8..12]) as i32); + let mut x13 = _mm512_set1_epi32(load_u32_le(&nonce[0..4]).cast_signed()); + let mut x14 = _mm512_set1_epi32(load_u32_le(&nonce[4..8]).cast_signed()); + let mut x15 = _mm512_set1_epi32(load_u32_le(&nonce[8..12]).cast_signed()); let o0 = x0; let o1 = x1; @@ -108,7 +127,6 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: // 16 blocks) to block-major (each register = one complete 64-byte block). // // Stage 1: 32-bit interleave — pairwise unpack adjacent state-word registers. - // SAFETY: AVX-512 intrinsics are valid under the enclosing target_feature. let s1_0 = _mm512_unpacklo_epi32(x0, x1); let s1_1 = _mm512_unpackhi_epi32(x0, x1); let s1_2 = _mm512_unpacklo_epi32(x2, x3); @@ -215,7 +233,7 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: xor_block(ptr, 15, blk15); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(COUNTERS_PER_BATCH); } let remainder = batches.into_remainder(); @@ -224,7 +242,7 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: // SAFETY: AVX-512-ready CPUs provide the SSSE3 instructions used by the // 4-block tail kernel, and `chunk` is exactly 4 ChaCha20 blocks. unsafe { x86_ssse3_x4::xor_blocks(key, counter, nonce, chunk) }; - counter = counter.wrapping_add(x86_ssse3_x4::BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(x86_ssse3_x4::COUNTERS_PER_BATCH); } let remainder = x4_batches.into_remainder(); @@ -234,11 +252,18 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: } /// Load 64 bytes of plaintext at block offset `idx`, XOR with keystream block, store. +/// +/// # Safety +/// +/// The caller must ensure that AVX512F is available, `buf` is valid for exclusive access to 1,024 initialized bytes, +/// and `idx` is less than 16. The buffer need not be 64-byte aligned. #[inline(always)] unsafe fn xor_block(buf: *mut u8, idx: usize, keystream: __m512i) { // SAFETY: caller guarantees `buf` points to a 1024-byte chunk and `idx < 16`. unsafe { - let p = buf.add(idx.strict_mul(BLOCK_SIZE)).cast::<__m512i>(); + let p = core::ptr::NonNull::new_unchecked(buf.add(idx.strict_mul(BLOCK_SIZE))) + .cast::<__m512i>() + .as_ptr(); let plaintext = _mm512_loadu_si512(p); _mm512_storeu_si512(p, _mm512_xor_si512(plaintext, keystream)); } diff --git a/src/aead/chacha20/x86_64_ssse3_x4.rs b/src/aead/chacha20/x86_64_ssse3_x4.rs index 7ac1cce2..acfb8b9e 100644 --- a/src/aead/chacha20/x86_64_ssse3_x4.rs +++ b/src/aead/chacha20/x86_64_ssse3_x4.rs @@ -7,7 +7,14 @@ use core::arch::x86_64::{ use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le}; pub(super) const BLOCKS_PER_BATCH: usize = 4; - +pub(super) const COUNTERS_PER_BATCH: u32 = 4; + +/// XOR four consecutive ChaCha20 blocks with the SSSE3 four-way kernel. +/// +/// # Safety +/// +/// The caller must ensure that SSSE3 and AVX are available, `chunk` is exactly four blocks, and +/// `initial_counter + 3` fits in `u32`. #[inline] pub(super) unsafe fn xor_blocks( key: &[u8; KEY_SIZE], @@ -16,37 +23,45 @@ pub(super) unsafe fn xor_blocks( chunk: &mut [u8], ) { debug_assert_eq!(chunk.len(), BLOCK_SIZE * BLOCKS_PER_BATCH); + debug_assert!(initial_counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); // SAFETY: callers only reach this helper from x86 SIMD backends whose - // dispatch gates guarantee AVX/SSSE3-capable hardware. + // dispatch or forced-backend contracts guarantee AVX/SSSE3-capable hardware; the assertions above restate the + // exact block and counter bounds. unsafe { xor_blocks_impl(key, initial_counter, nonce, chunk) } } +/// Generate and XOR four consecutive ChaCha20 blocks. +/// +/// # Safety +/// +/// The caller must ensure that SSSE3 and AVX are available, `chunk` is exactly four blocks, and +/// `initial_counter + 3` fits in `u32`. #[target_feature(enable = "ssse3,avx")] unsafe fn xor_blocks_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], chunk: &mut [u8]) { let rot16 = _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); let rot8 = _mm_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3); - let mut x0 = _mm_set1_epi32(0x6170_7865u32 as i32); - let mut x1 = _mm_set1_epi32(0x3320_646eu32 as i32); - let mut x2 = _mm_set1_epi32(0x7962_2d32u32 as i32); - let mut x3 = _mm_set1_epi32(0x6b20_6574u32 as i32); - let mut x4 = _mm_set1_epi32(load_u32_le(&key[0..4]) as i32); - let mut x5 = _mm_set1_epi32(load_u32_le(&key[4..8]) as i32); - let mut x6 = _mm_set1_epi32(load_u32_le(&key[8..12]) as i32); - let mut x7 = _mm_set1_epi32(load_u32_le(&key[12..16]) as i32); - let mut x8 = _mm_set1_epi32(load_u32_le(&key[16..20]) as i32); - let mut x9 = _mm_set1_epi32(load_u32_le(&key[20..24]) as i32); - let mut x10 = _mm_set1_epi32(load_u32_le(&key[24..28]) as i32); - let mut x11 = _mm_set1_epi32(load_u32_le(&key[28..32]) as i32); + let mut x0 = _mm_set1_epi32(0x6170_7865u32.cast_signed()); + let mut x1 = _mm_set1_epi32(0x3320_646eu32.cast_signed()); + let mut x2 = _mm_set1_epi32(0x7962_2d32u32.cast_signed()); + let mut x3 = _mm_set1_epi32(0x6b20_6574u32.cast_signed()); + let mut x4 = _mm_set1_epi32(load_u32_le(&key[0..4]).cast_signed()); + let mut x5 = _mm_set1_epi32(load_u32_le(&key[4..8]).cast_signed()); + let mut x6 = _mm_set1_epi32(load_u32_le(&key[8..12]).cast_signed()); + let mut x7 = _mm_set1_epi32(load_u32_le(&key[12..16]).cast_signed()); + let mut x8 = _mm_set1_epi32(load_u32_le(&key[16..20]).cast_signed()); + let mut x9 = _mm_set1_epi32(load_u32_le(&key[20..24]).cast_signed()); + let mut x10 = _mm_set1_epi32(load_u32_le(&key[24..28]).cast_signed()); + let mut x11 = _mm_set1_epi32(load_u32_le(&key[28..32]).cast_signed()); let mut x12 = _mm_setr_epi32( - initial_counter as i32, - initial_counter.wrapping_add(1) as i32, - initial_counter.wrapping_add(2) as i32, - initial_counter.wrapping_add(3) as i32, + initial_counter.cast_signed(), + initial_counter.wrapping_add(1).cast_signed(), + initial_counter.wrapping_add(2).cast_signed(), + initial_counter.wrapping_add(3).cast_signed(), ); - let mut x13 = _mm_set1_epi32(load_u32_le(&nonce[0..4]) as i32); - let mut x14 = _mm_set1_epi32(load_u32_le(&nonce[4..8]) as i32); - let mut x15 = _mm_set1_epi32(load_u32_le(&nonce[8..12]) as i32); + let mut x13 = _mm_set1_epi32(load_u32_le(&nonce[0..4]).cast_signed()); + let mut x14 = _mm_set1_epi32(load_u32_le(&nonce[4..8]).cast_signed()); + let mut x15 = _mm_set1_epi32(load_u32_le(&nonce[8..12]).cast_signed()); let o0 = x0; let o1 = x1; @@ -129,6 +144,11 @@ fn transpose_words(w0: __m128i, w1: __m128i, w2: __m128i, w3: __m128i) -> [__m12 } } +/// XOR one generated ChaCha20 block into a selected block of a four-block buffer. +/// +/// # Safety +/// +/// `buf` must be valid for exclusive access to 256 initialized bytes, and `idx` must be less than four. #[inline(always)] unsafe fn xor_block(buf: *mut u8, idx: usize, w03: __m128i, w47: __m128i, w811: __m128i, w1215: __m128i) { // SAFETY: caller guarantees `buf` points to four full ChaCha20 blocks and @@ -142,6 +162,11 @@ unsafe fn xor_block(buf: *mut u8, idx: usize, w03: __m128i, w47: __m128i, w811: } } +/// XOR 16 keystream bytes into an in-place buffer segment. +/// +/// # Safety +/// +/// `ptr` must be valid for reading and exclusively writing 16 initialized bytes. It need not be aligned. #[inline(always)] unsafe fn xor_store(ptr: *mut u8, keystream: __m128i) { // SAFETY: caller guarantees `ptr..ptr+16` is in bounds for unaligned access. diff --git a/src/aead/chacha20poly1305.rs b/src/aead/chacha20poly1305.rs index 0309998f..5c627afb 100644 --- a/src/aead/chacha20poly1305.rs +++ b/src/aead/chacha20poly1305.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! ChaCha20-Poly1305 public AEAD surface. use core::fmt; @@ -20,7 +18,9 @@ const POWER_SHORT_FAST_MAX: usize = chacha20::BLOCK_SIZE; #[cfg(target_arch = "aarch64")] const AARCH64_INTERLEAVED_MIN: usize = 1024; #[cfg(target_arch = "aarch64")] -const AARCH64_INTERLEAVED_CHUNK: usize = 1024 * 1024; +const AARCH64_INTERLEAVED_BLOCKS: u32 = 16 * 1024; +#[cfg(target_arch = "aarch64")] +const AARCH64_INTERLEAVED_CHUNK: usize = (AARCH64_INTERLEAVED_BLOCKS as usize) * chacha20::BLOCK_SIZE; #[cfg(all( target_arch = "x86_64", target_os = "linux", @@ -323,7 +323,9 @@ impl ChaCha20Poly1305 { return None; } - let tag = x86_64_asm::seal_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer); + // SAFETY: the diagnostic entry validated the ChaCha20 length bound; the guards above prove a nonempty buffer and + // AVX2+BMI2 availability. + let tag = unsafe { x86_64_asm::seal_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer) }; Some(Ok(ChaCha20Poly1305Tag::from_bytes(tag))) } @@ -348,7 +350,9 @@ impl ChaCha20Poly1305 { return None; } - let expected = x86_64_asm::open_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer); + // SAFETY: the diagnostic entry validated the ChaCha20 length bound; the guards above prove a nonempty buffer and + // AVX2+BMI2 availability. + let expected = unsafe { x86_64_asm::open_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer) }; if !ct::fixed_eq(&expected, tag.as_bytes()).declassify() { ct::zeroize(buffer); return Some(Err(OpenError::verification())); @@ -377,7 +381,9 @@ impl ChaCha20Poly1305 { return None; } - let tag = x86_64_asm::seal_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer); + // SAFETY: the public AEAD entry validated the ChaCha20 length bound, and the recommendation gate proves a + // nonempty buffer plus AVX2+BMI2 availability. + let tag = unsafe { x86_64_asm::seal_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer) }; Some(Ok(ChaCha20Poly1305Tag::from_bytes(tag))) } @@ -403,7 +409,9 @@ impl ChaCha20Poly1305 { return None; } - let expected = x86_64_asm::open_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer); + // SAFETY: the public AEAD entry validated the ChaCha20 length bound, and the recommendation gate proves a + // nonempty buffer plus AVX2+BMI2 availability. + let expected = unsafe { x86_64_asm::open_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer) }; if !ct::fixed_eq(&expected, tag.as_bytes()).declassify() { ct::zeroize(buffer); return Some(Err(OpenError::verification())); @@ -490,14 +498,20 @@ impl ChaCha20Poly1305 { let mut counter = 1u32; let mut chunks = buffer.chunks_exact_mut(AARCH64_INTERLEAVED_CHUNK); for chunk in &mut chunks { - chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), chunk); + // SAFETY: the capability gate proves NEON. Every caller reaches this private helper after the public or + // diagnostic length bound, or from a same-module test with a bounded buffer; the iterator and exact counter + // advance preserve that whole-buffer bound for this segment. + unsafe { chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), chunk) }; authenticator.update_padded_segment(chunk); - counter = counter.wrapping_add((AARCH64_INTERLEAVED_CHUNK / chacha20::BLOCK_SIZE) as u32); + counter = counter.wrapping_add(AARCH64_INTERLEAVED_BLOCKS); } let remainder = chunks.into_remainder(); if !remainder.is_empty() { - chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), remainder); + // SAFETY: the capability gate proves NEON. Every caller reaches this private helper after the public or + // diagnostic length bound, or from a same-module test with a bounded buffer; `counter` tracks the preceding + // full chunks exactly, so the final remainder cannot exhaust the counter range. + unsafe { chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), remainder) }; authenticator.update_padded_segment(remainder); } @@ -542,14 +556,20 @@ impl ChaCha20Poly1305 { let mut chunks = buffer.chunks_exact_mut(AARCH64_INTERLEAVED_CHUNK); for chunk in &mut chunks { authenticator.update_padded_segment(chunk); - chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), chunk); - counter = counter.wrapping_add((AARCH64_INTERLEAVED_CHUNK / chacha20::BLOCK_SIZE) as u32); + // SAFETY: the capability gate proves NEON. Every caller reaches this private helper after the public or + // diagnostic length bound, or from a same-module test with a bounded buffer; the iterator and exact counter + // advance preserve that whole-buffer bound for this segment. + unsafe { chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), chunk) }; + counter = counter.wrapping_add(AARCH64_INTERLEAVED_BLOCKS); } let remainder = chunks.into_remainder(); if !remainder.is_empty() { authenticator.update_padded_segment(remainder); - chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), remainder); + // SAFETY: the capability gate proves NEON. Every caller reaches this private helper after the public or + // diagnostic length bound, or from a same-module test with a bounded buffer; `counter` tracks the preceding + // full chunks exactly, so the final remainder cannot exhaust the counter range. + unsafe { chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), remainder) }; } let expected = authenticator.finalize(lengths); @@ -622,6 +642,10 @@ impl ChaCha20Poly1305 { } } +/// Encrypts in place without using a platform-specific integrated ChaCha20-Poly1305 assembly entrypoint. +/// +/// Lower-level ChaCha20 and Poly1305 dispatch remains enabled. Returns an error when the input lengths exceed the +/// supported limits. #[cfg(feature = "diag")] pub fn diag_chacha20poly1305_encrypt_in_place_owned( cipher: &ChaCha20Poly1305, @@ -634,6 +658,9 @@ pub fn diag_chacha20poly1305_encrypt_in_place_owned( } #[cfg(all(feature = "diag", target_arch = "x86_64", target_os = "linux"))] +/// Encrypts in place through the Linux x86-64 assembly entrypoint when that entrypoint is available. +/// +/// Returns `None` when the current CPU cannot execute the assembly backend. pub fn diag_chacha20poly1305_encrypt_in_place_x86_64_asm( cipher: &ChaCha20Poly1305, nonce: &Nonce96, @@ -648,6 +675,9 @@ pub fn diag_chacha20poly1305_encrypt_in_place_x86_64_asm( } #[cfg(all(feature = "diag", target_arch = "x86_64", target_os = "linux"))] +/// Authenticates and decrypts in place through the Linux x86-64 assembly entrypoint when it is available. +/// +/// Returns `None` when the current CPU cannot execute the assembly backend. pub fn diag_chacha20poly1305_decrypt_in_place_x86_64_asm( cipher: &ChaCha20Poly1305, nonce: &Nonce96, @@ -662,6 +692,11 @@ pub fn diag_chacha20poly1305_decrypt_in_place_x86_64_asm( cipher.decrypt_in_place_asm_x86_64_forced(nonce, aad, buffer, tag) } +/// Authenticates and decrypts in place without using a platform-specific integrated ChaCha20-Poly1305 assembly +/// entrypoint. +/// +/// Lower-level ChaCha20 and Poly1305 dispatch remains enabled. Authentication failure zeroes `buffer` and returns an +/// opaque verification error; unsupported input lengths also return an error. #[cfg(feature = "diag")] pub fn diag_chacha20poly1305_decrypt_in_place_owned( cipher: &ChaCha20Poly1305, @@ -803,8 +838,12 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); let mut buf = *b"hello chacha"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); - cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &tag).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("valid input must encrypt"); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buf, &tag) + .expect("matching nonce, AAD, and tag must decrypt"); assert_eq!(&buf, b"hello chacha"); } @@ -815,7 +854,9 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); let mut buf = *b"nonce test"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("valid input must encrypt"); let wrong_nonce = Nonce96::from_bytes([0x08u8; 12]); let result = cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag); @@ -829,7 +870,9 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); let mut buf = *b"zero me on failure"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("valid input must encrypt"); let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0xFF; @@ -900,14 +943,15 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); for aad_len in [0usize, 1, 14, 15, 16, 17, 31, 32, 33, 63, 64] { - let aad = (0..aad_len) - .map(|index| 0xa7u8.wrapping_add((index as u8).wrapping_mul(7))) + let aad = (0u8..) + .take(aad_len) + .map(|index| 0xa7u8.wrapping_add(index.wrapping_mul(7))) .collect::>(); let mut ciphertext = Vec::new(); let expected_tag = cipher .encrypt_in_place_owned_unchecked(&nonce, &aad, &mut ciphertext) - .unwrap(); + .expect("owned path must produce an empty-text tag"); let actual = cipher.decrypt_empty_text_fast(&nonce, &aad, &expected_tag); if aad_len > SMALL_AAD_FAST_MAX { @@ -918,9 +962,10 @@ mod tests { continue; } - actual - .expect("empty decrypt fast path must apply inside its configured gate") - .unwrap(); + assert_eq!( + actual.expect("empty decrypt fast path must apply inside its configured gate"), + Ok(()) + ); let mut bad_tag = expected_tag.to_bytes(); bad_tag[0] ^= 0x80; @@ -961,7 +1006,9 @@ mod tests { .encrypt_in_place_owned_unchecked(&nonce, &aad, &mut ciphertext) .unwrap(); let mut actual = ciphertext.clone(); - let actual_tag = x86_64_asm::open_in_place(key.as_bytes(), nonce.as_bytes(), &aad, &mut actual); + // SAFETY: the test returned unless AVX2+BMI2 are available; every selected ciphertext is nonempty and well + // below ChaCha20's 2^32-block limit. + let actual_tag = unsafe { x86_64_asm::open_in_place(key.as_bytes(), nonce.as_bytes(), &aad, &mut actual) }; assert_eq!( actual, plaintext, @@ -984,13 +1031,15 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); for plaintext_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65] { - let plaintext = (0..plaintext_len) - .map(|index| 0x51u8.wrapping_add((index as u8).wrapping_mul(13))) + let plaintext = (0u8..) + .take(plaintext_len) + .map(|index| 0x51u8.wrapping_add(index.wrapping_mul(13))) .collect::>(); for aad_len in [0usize, 1, 14, 15, 16, 17, 31, 32, 33, 63, 64] { - let aad = (0..aad_len) - .map(|index| 0xa7u8.wrapping_add((index as u8).wrapping_mul(7))) + let aad = (0u8..) + .take(aad_len) + .map(|index| 0xa7u8.wrapping_add(index.wrapping_mul(7))) .collect::>(); let mut actual = plaintext.clone(); @@ -1008,10 +1057,10 @@ mod tests { let mut expected = plaintext.clone(); let expected_tag = cipher .encrypt_in_place_owned_unchecked(&nonce, &aad, &mut expected) - .unwrap(); + .expect("bounded test input must encrypt through the owned path"); let actual_tag = actual_tag .expect("Power short fast path must apply inside its configured gate") - .unwrap(); + .expect("bounded test input must encrypt through the Power fast path"); assert_eq!( actual, expected, @@ -1033,19 +1082,21 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); for plaintext_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65] { - let plaintext = (0..plaintext_len) - .map(|index| 0x51u8.wrapping_add((index as u8).wrapping_mul(13))) + let plaintext = (0u8..) + .take(plaintext_len) + .map(|index| 0x51u8.wrapping_add(index.wrapping_mul(13))) .collect::>(); for aad_len in [0usize, 1, 14, 15, 16, 17, 31, 32, 33, 63, 64] { - let aad = (0..aad_len) - .map(|index| 0xa7u8.wrapping_add((index as u8).wrapping_mul(7))) + let aad = (0u8..) + .take(aad_len) + .map(|index| 0xa7u8.wrapping_add(index.wrapping_mul(7))) .collect::>(); let mut ciphertext = plaintext.clone(); let tag = cipher .encrypt_in_place_owned_unchecked(&nonce, &aad, &mut ciphertext) - .unwrap(); + .expect("bounded test input must encrypt through the owned path"); let mut actual = ciphertext.clone(); let actual_result = cipher.decrypt_short_text_power_fast(&nonce, &aad, &mut actual, &tag); @@ -1059,9 +1110,10 @@ mod tests { continue; } - actual_result - .expect("Power short decrypt fast path must apply inside its configured gate") - .unwrap(); + assert_eq!( + actual_result.expect("Power short decrypt fast path must apply inside its configured gate"), + Ok(()) + ); assert_eq!( actual, plaintext, "Power short plaintext mismatch plaintext_len={plaintext_len} aad_len={aad_len}" @@ -1091,7 +1143,9 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); let mut buf = *b"aad test"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buf) + .expect("valid input must encrypt"); let result = cipher.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag); assert!(result.is_err()); diff --git a/src/aead/chacha20poly1305/aarch64_asm.rs b/src/aead/chacha20poly1305/aarch64_asm.rs index c2d1efc5..85b7471e 100644 --- a/src/aead/chacha20poly1305/aarch64_asm.rs +++ b/src/aead/chacha20poly1305/aarch64_asm.rs @@ -4,8 +4,6 @@ //! ChaCha20-Poly1305 assembly. This Rust module owns the ABI boundary and //! keeps runtime feature dispatch in the parent AEAD implementation. -#![allow(unsafe_code)] - use core::{arch::global_asm, mem}; use super::KEY_SIZE; @@ -120,7 +118,10 @@ pub(super) fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], // empty slices because the length is zero. // 4. `extra_ciphertext` points at 16 initialized bytes and `extra_ciphertext_len` is zero, matching // the AWS-LC seal ABI for callers without extra trailing ciphertext. - // 5. `data` is 16-byte aligned and matches the assembly input/output union layout. + // 5. `data` is 16-byte aligned and matches the assembly input/output union layout; the assembly initializes all + // 16 bytes of `data.out.tag` before the union field is read. + // 6. After that write, the tag replaces the first 16 copied key bytes; the second half remains initialized key + // material and is volatile-zeroed before the stack allocation expires. unsafe { #[cfg(target_os = "macos")] rscrypto_chacha20_poly1305_seal_aarch64_apple_darwin( @@ -129,7 +130,7 @@ pub(super) fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer.len(), aad.as_ptr(), aad.len(), - &mut data, + core::ptr::from_mut(&mut data), ); #[cfg(target_os = "linux")] rscrypto_chacha20_poly1305_seal_aarch64( @@ -138,8 +139,9 @@ pub(super) fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer.len(), aad.as_ptr(), aad.len(), - &mut data, + core::ptr::from_mut(&mut data), ); + crate::traits::ct::zeroize(&mut data.input.key[16..]); data.out.tag } } @@ -161,7 +163,10 @@ pub(super) fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], // the assembly routine supports in-place open, matching the AWS-LC ABI. // 3. `aad.as_ptr()` is valid for `aad.len()` bytes, including the conventional dangling pointer for // empty slices because the length is zero. - // 4. `data` is 16-byte aligned and matches the assembly input/output union layout. + // 4. `data` is 16-byte aligned and matches the assembly input/output union layout; the assembly initializes all + // 16 bytes of `data.out.tag` before the union field is read. + // 5. After that write, the tag replaces the first 16 copied key bytes; the second half remains initialized key + // material and is volatile-zeroed before the stack allocation expires. unsafe { #[cfg(target_os = "macos")] rscrypto_chacha20_poly1305_open_aarch64_apple_darwin( @@ -170,7 +175,7 @@ pub(super) fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer.len(), aad.as_ptr(), aad.len(), - &mut data, + core::ptr::from_mut(&mut data), ); #[cfg(target_os = "linux")] rscrypto_chacha20_poly1305_open_aarch64( @@ -179,8 +184,9 @@ pub(super) fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer.len(), aad.as_ptr(), aad.len(), - &mut data, + core::ptr::from_mut(&mut data), ); + crate::traits::ct::zeroize(&mut data.input.key[16..]); data.out.tag } } diff --git a/src/aead/chacha20poly1305/x86_64_asm.rs b/src/aead/chacha20poly1305/x86_64_asm.rs index bbc2351b..55ac6124 100644 --- a/src/aead/chacha20poly1305/x86_64_asm.rs +++ b/src/aead/chacha20poly1305/x86_64_asm.rs @@ -4,8 +4,6 @@ //! ChaCha20-Poly1305 assembly. This Rust module owns the ABI boundary and //! keeps runtime feature dispatch in the parent AEAD implementation. -#![allow(unsafe_code)] - use core::{arch::global_asm, mem}; use super::KEY_SIZE; @@ -76,8 +74,14 @@ unsafe extern "C" { ); } +/// Seal one nonempty buffer with the integrated x86-64 assembly kernel. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 and BMI2 are available and that `buffer` is nonempty and within ChaCha20's +/// 2³²-block limit. #[inline] -pub(super) fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer: &mut [u8]) -> [u8; 16] { +pub(super) unsafe fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer: &mut [u8]) -> [u8; 16] { debug_assert!(!buffer.is_empty()); let extra_ciphertext = [0u8; 16]; @@ -99,7 +103,11 @@ pub(super) fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], // empty slices because the length is zero. // 4. `extra_ciphertext` points at 16 initialized bytes and `extra_ciphertext_len` is zero, matching // the AWS-LC seal ABI for callers without extra trailing ciphertext. - // 5. `data` is 16-byte aligned and matches the assembly input/output union layout. + // 5. `data` is 16-byte aligned and matches the assembly input/output union layout; `from_mut` preserves that + // allocation's provenance and exclusive writability for the call, which initializes all 16 tag bytes before + // the union output is read. + // 6. The tag replaces the first 16 copied key bytes; the second half remains initialized key material and is + // volatile-zeroed before the stack allocation expires. unsafe { rscrypto_chacha20_poly1305_seal_x86_64( buffer.as_mut_ptr(), @@ -107,14 +115,21 @@ pub(super) fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer.len(), aad.as_ptr(), aad.len(), - &mut data, + core::ptr::from_mut(&mut data), ); + crate::traits::ct::zeroize(&mut data.input.key[16..]); data.out.tag } } +/// Open one nonempty buffer with the integrated x86-64 assembly kernel. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 and BMI2 are available and that `buffer` is nonempty and within ChaCha20's +/// 2³²-block limit. #[inline] -pub(super) fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer: &mut [u8]) -> [u8; 16] { +pub(super) unsafe fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer: &mut [u8]) -> [u8; 16] { debug_assert!(!buffer.is_empty()); let mut data = OpenData { @@ -131,7 +146,11 @@ pub(super) fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], // the assembly routine supports in-place open, matching the AWS-LC ABI. // 3. `aad.as_ptr()` is valid for `aad.len()` bytes, including the conventional dangling pointer for // empty slices because the length is zero. - // 4. `data` is 16-byte aligned and matches the assembly input/output union layout. + // 4. `data` is 16-byte aligned and matches the assembly input/output union layout; `from_mut` preserves that + // allocation's provenance and exclusive writability for the call, which initializes all 16 tag bytes before + // the union output is read. + // 5. The tag replaces the first 16 copied key bytes; the second half remains initialized key material and is + // volatile-zeroed before the stack allocation expires. unsafe { rscrypto_chacha20_poly1305_open_x86_64( buffer.as_mut_ptr(), @@ -139,8 +158,9 @@ pub(super) fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer.len(), aad.as_ptr(), aad.len(), - &mut data, + core::ptr::from_mut(&mut data), ); + crate::traits::ct::zeroize(&mut data.input.key[16..]); data.out.tag } } diff --git a/src/aead/ghash.rs b/src/aead/ghash.rs index 77ef35e6..55470c44 100644 --- a/src/aead/ghash.rs +++ b/src/aead/ghash.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! Fixed-schedule, table-free GHASH universal hash (NIST SP 800-38D). //! //! Generated-code timing claims remain configuration- and @@ -52,6 +50,7 @@ pub(crate) fn h_to_polyval(h_bytes: &[u8; KEY_SIZE]) -> u128 { mul_x_polyval(h) } +/// Computes one GHASH block with the portable POLYVAL-domain reduction. #[cfg(feature = "diag")] #[must_use] pub fn diag_ghash_block_portable(h_bytes: &[u8; KEY_SIZE], block: &[u8; KEY_SIZE]) -> [u8; KEY_SIZE] { @@ -129,10 +128,11 @@ impl Ghash { #[cfg(test)] impl Drop for Ghash { fn drop(&mut self) { - // SAFETY: self.acc/self.h are valid, aligned, dereferenceable pointers to initialized memory. + // SAFETY: the raw pointers address initialized, aligned `u128` fields owned + // exclusively by `self` for the duration of `drop`. unsafe { - core::ptr::write_volatile(&mut self.acc, 0); - core::ptr::write_volatile(&mut self.h, 0); + core::ptr::write_volatile(&raw mut self.acc, 0); + core::ptr::write_volatile(&raw mut self.h, 0); } core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } @@ -144,6 +144,8 @@ impl Drop for Ghash { mod tests { use super::*; + const TEST_H: [u8; 16] = 0x66e9_4bd4_ef8a_2c3b_884c_fa59_ca34_2b2eu128.to_be_bytes(); + /// GHASH with empty input should return zero. #[test] fn ghash_empty() { @@ -165,26 +167,26 @@ mod tests { /// Verify update_padded matches manual block-by-block. #[test] fn ghash_padded_matches_manual() { - let h = hex_to_16("66e94bd4ef8a2c3b884cfa59ca342b2e"); let data = b"Hello, World! This is test data for GHASH padding."; // Manual: split into 16-byte blocks, pad last one. - let mut manual = Ghash::new(&h); - let mut offset = 0; - while offset + 16 <= data.len() { - let block: [u8; 16] = data[offset..offset + 16].try_into().unwrap(); + let mut manual = Ghash::new(&TEST_H); + let mut chunks = data.chunks_exact(BLOCK_SIZE); + for chunk in chunks.by_ref() { + let mut block = [0u8; BLOCK_SIZE]; + block.copy_from_slice(chunk); manual.update_block(&block); - offset += 16; } - if offset < data.len() { - let mut block = [0u8; 16]; - block[..data.len() - offset].copy_from_slice(&data[offset..]); + let remainder = chunks.remainder(); + if !remainder.is_empty() { + let mut block = [0u8; BLOCK_SIZE]; + block[..remainder.len()].copy_from_slice(remainder); manual.update_block(&block); } let manual_result = manual.finalize(); // Padded API. - let mut padded = Ghash::new(&h); + let mut padded = Ghash::new(&TEST_H); padded.update_padded(data); let padded_result = padded.finalize(); @@ -213,14 +215,4 @@ mod tests { "mulX(x^127) should reduce to feedback polynomial" ); } - - fn hex_to_16(hex: &str) -> [u8; 16] { - let mut out = [0u8; 16]; - let mut i = 0; - while i < 16 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - i = i.strict_add(1); - } - out - } } diff --git a/src/aead/introspect.rs b/src/aead/introspect.rs index fe12e5c5..b350c8f8 100644 --- a/src/aead/introspect.rs +++ b/src/aead/introspect.rs @@ -18,8 +18,7 @@ feature = "aes-gcm-siv", feature = "chacha20poly1305", feature = "xchacha20poly1305", - feature = "aegis256", - feature = "ascon-aead" + feature = "aegis256" ))] use crate::aead::targets::{AeadPrimitive, select_backend}; pub use crate::platform::DispatchInfo; @@ -29,8 +28,7 @@ pub use crate::platform::DispatchInfo; feature = "aes-gcm-siv", feature = "chacha20poly1305", feature = "xchacha20poly1305", - feature = "aegis256", - feature = "ascon-aead" + feature = "aegis256" ))] #[inline] fn backend_for(primitive: AeadPrimitive) -> &'static str { @@ -98,7 +96,10 @@ pub fn aegis256_backend() -> &'static str { #[inline] #[must_use] pub fn ascon_aead128_backend() -> &'static str { - backend_for(AeadPrimitive::AsconAead128) + match crate::platform::arch() { + crate::platform::Arch::Wasm32 | crate::platform::Arch::Wasm64 => "wasm32/portable", + _ => "portable", + } } #[cfg(test)] diff --git a/src/aead/mod.rs b/src/aead/mod.rs index 0ed370d8..366faeee 100644 --- a/src/aead/mod.rs +++ b/src/aead/mod.rs @@ -34,10 +34,10 @@ //! ```toml //! [dependencies] //! # ChaCha20-Poly1305 only -//! rscrypto = { version = "0.5.0", default-features = false, features = ["chacha20poly1305"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["chacha20poly1305"] } //! //! # All AEADs -//! rscrypto = { version = "0.5.0", default-features = false, features = ["aead"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["aead"] } //! ``` //! //! # API Conventions @@ -73,14 +73,6 @@ mod aegis256; all(feature = "aegis256", target_arch = "riscv64"), all(feature = "aegis256", test), ))] -#[cfg_attr( - all( - feature = "aegis256", - target_arch = "riscv64", - not(any(feature = "aes-gcm", feature = "aes-gcm-siv")) - ), - allow(dead_code) -)] mod aes; #[cfg(feature = "aes-gcm")] mod aes128gcm; @@ -95,6 +87,42 @@ mod aes256gcmsiv; all(target_arch = "riscv64", any(feature = "aes-gcm", feature = "aes-gcm-siv")) ))] mod aes_round; +#[cfg(all(test, any(feature = "aes-gcm", feature = "aes-gcm-siv", feature = "aegis256")))] +mod test_vectors { + use alloc::{vec, vec::Vec}; + + #[track_caller] + pub(super) fn hex_array(hex: &str) -> [u8; N] { + let mut out = [0u8; N]; + crate::hex::from_hex(hex, &mut out).expect("AEAD test vector must contain valid hexadecimal"); + out + } + + #[track_caller] + #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] + pub(super) fn hex12(hex: &str) -> [u8; 12] { + hex_array(hex) + } + + #[track_caller] + #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] + pub(super) fn hex16(hex: &str) -> [u8; 16] { + hex_array(hex) + } + + #[track_caller] + #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] + pub(super) fn hex32(hex: &str) -> [u8; 32] { + hex_array(hex) + } + + #[track_caller] + pub(super) fn hex_vec(hex: &str) -> Vec { + let mut out = vec![0u8; hex.as_bytes().chunks_exact(2).len()]; + crate::hex::from_hex(hex, &mut out).expect("AEAD test vector must contain valid hexadecimal"); + out + } +} #[cfg(feature = "ascon-aead")] mod ascon128; #[cfg(any(feature = "chacha20poly1305", feature = "xchacha20poly1305"))] @@ -111,6 +139,33 @@ mod nonce_counter; mod poly1305; #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] mod polyval; +#[cfg(any( + feature = "aes-gcm", + feature = "chacha20poly1305", + feature = "xchacha20poly1305", + all( + feature = "aes-gcm-siv", + any( + feature = "diag", + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ), + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) +))] mod targets; #[cfg(feature = "xchacha20poly1305")] mod xchacha20poly1305; @@ -595,36 +650,40 @@ impl From for OpenError { } } -#[cfg_attr( - not(any( - feature = "aes-gcm", - feature = "aes-gcm-siv", - feature = "chacha20poly1305", - feature = "xchacha20poly1305" - )), - allow(dead_code) -)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "xchacha20poly1305" +))] pub(crate) struct LengthOverflow; const _: () = assert!(usize::BITS <= u64::BITS); -#[cfg_attr( - not(any( - feature = "aes-gcm", - feature = "aes-gcm-siv", - feature = "chacha20poly1305", - feature = "xchacha20poly1305" - )), - allow(dead_code) -)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "xchacha20poly1305" +))] pub(crate) struct AeadByteLengths { aad: u64, text: u64, } +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "xchacha20poly1305" +))] impl AeadByteLengths { + #[cfg(any(feature = "aes-gcm-siv", feature = "chacha20poly1305", feature = "xchacha20poly1305"))] #[inline] pub(crate) const fn from_usize(aad_len: usize, text_len: usize) -> Self { Self { @@ -633,19 +692,13 @@ impl AeadByteLengths { } } - #[cfg_attr( - not(any(feature = "chacha20poly1305", feature = "xchacha20poly1305")), - allow(dead_code) - )] + #[cfg(any(feature = "chacha20poly1305", feature = "xchacha20poly1305"))] #[inline] pub(crate) fn try_new(aad_len: usize, text_len: usize) -> Result { Ok(Self::from_usize(aad_len, text_len)) } - #[cfg_attr( - not(any(feature = "aes-gcm", feature = "aes-gcm-siv", feature = "aegis256")), - allow(dead_code) - )] + #[cfg(any(feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] #[inline] pub(crate) fn try_new_bit_lengths(aad_len: usize, text_len: usize) -> Result { const MAX_BIT_ENCODED_BYTES: u64 = u64::MAX / 8; @@ -655,10 +708,7 @@ impl AeadByteLengths { }) } - #[cfg_attr( - not(any(feature = "chacha20poly1305", feature = "xchacha20poly1305")), - allow(dead_code) - )] + #[cfg(any(feature = "chacha20poly1305", feature = "xchacha20poly1305"))] #[inline] pub(crate) fn to_le_bytes_block(self) -> [u8; 16] { let mut block = [0u8; 16]; @@ -667,7 +717,7 @@ impl AeadByteLengths { block } - #[cfg_attr(not(feature = "aes-gcm-siv"), allow(dead_code))] + #[cfg(feature = "aes-gcm-siv")] #[inline] pub(crate) fn to_le_bits_block(self) -> [u8; 16] { let aad_bits = self.aad.strict_mul(8); @@ -678,7 +728,7 @@ impl AeadByteLengths { block } - #[cfg_attr(not(feature = "aes-gcm"), allow(dead_code))] + #[cfg(feature = "aes-gcm")] #[inline] pub(crate) fn to_be_bits_block(self) -> [u8; 16] { let aad_bits = self.aad.strict_mul(8); @@ -689,45 +739,38 @@ impl AeadByteLengths { block } - #[cfg_attr( - not(all( - target_arch = "x86_64", - any(feature = "chacha20poly1305", feature = "xchacha20poly1305") - )), - allow(dead_code) - )] #[inline] + #[cfg(all( + any(feature = "chacha20poly1305", feature = "xchacha20poly1305"), + any(test, target_arch = "x86_64", target_arch = "aarch64", target_arch = "riscv64") + ))] pub(crate) const fn total_at_least(self, minimum: u64) -> bool { if self.aad >= minimum { return true; } - self.text >= minimum - self.aad + self.text >= minimum.strict_sub(self.aad) } } -#[cfg_attr( - not(any( - feature = "aes-gcm", - feature = "aes-gcm-siv", - feature = "chacha20poly1305", - feature = "xchacha20poly1305" - )), - allow(dead_code) -)] +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "xchacha20poly1305" +))] #[inline] pub(crate) fn try_length_as_u64(len: usize) -> Result { u64::try_from(len).map_err(|_| LengthOverflow) } -#[cfg_attr( - not(any( - feature = "aes-gcm", - feature = "aes-gcm-siv", - feature = "chacha20poly1305", - feature = "xchacha20poly1305" - )), - allow(dead_code) -)] +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "xchacha20poly1305" +))] #[inline] pub(crate) fn try_bounded_length_as_u64(len: usize, max: u64) -> Result { let len = try_length_as_u64(len)?; @@ -737,47 +780,35 @@ pub(crate) fn try_bounded_length_as_u64(len: usize, max: u64) -> Result Result { try_bounded_length_as_u64(len, max).map_err(|_| SealError::too_large()) } -#[cfg_attr( - not(any(feature = "aes-gcm", feature = "aes-gcm-siv", feature = "aegis256")), - allow(dead_code) -)] +#[cfg(any(feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] #[inline] pub(crate) fn seal_bit_lengths(aad_len: usize, text_len: usize) -> Result { AeadByteLengths::try_new_bit_lengths(aad_len, text_len).map_err(|_| SealError::too_large()) } -#[cfg_attr( - not(any( - feature = "aes-gcm", - feature = "aes-gcm-siv", - feature = "chacha20poly1305", - feature = "xchacha20poly1305" - )), - allow(dead_code) -)] +#[cfg(any( + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "xchacha20poly1305" +))] #[inline] pub(crate) fn open_bounded_length_as_u64(len: usize, max: u64) -> Result { try_bounded_length_as_u64(len, max).map_err(|_| OpenError::too_large()) } -#[cfg_attr( - not(any(feature = "aes-gcm", feature = "aes-gcm-siv", feature = "aegis256")), - allow(dead_code) -)] +#[cfg(any(feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] #[inline] pub(crate) fn open_bit_lengths(aad_len: usize, text_len: usize) -> Result { AeadByteLengths::try_new_bit_lengths(aad_len, text_len).map_err(|_| OpenError::too_large()) @@ -787,7 +818,17 @@ pub(crate) fn open_bit_lengths(aad_len: usize, text_len: usize) -> Result` builds 96-bit nonces as: -//! -//! - 32-bit fixed prefix chosen by the caller -//! - 64-bit big-endian invocation counter -//! -//! This follows the deterministic IV shape from SP 800-38D and removes the -//! easiest nonce-reuse footgun from high-volume AES-GCM usage. -//! -//! ```rust -//! use rscrypto::{Aead, Aes256Gcm, Aes256GcmKey, aead::NonceCounter}; -//! -//! let cipher = Aes256Gcm::new(&Aes256GcmKey::from_bytes([0x42; 32])); -//! let mut counter = NonceCounter::::new(*b"sess"); -//! -//! let mut sealed = [0u8; 4 + Aes256Gcm::TAG_SIZE]; -//! let nonce = counter.encrypt(&cipher, b"hdr", b"data", &mut sealed)?; -//! -//! let mut opened = [0u8; 4]; -//! cipher.decrypt(&nonce, b"hdr", &sealed, &mut opened)?; -//! assert_eq!(&opened, b"data"); -//! # Ok::<(), Box>(()) -//! ``` - use core::{fmt, marker::PhantomData}; use super::{Aes128Gcm, Aes128GcmTag, Aes256Gcm, Aes256GcmTag, Nonce96, SealError}; @@ -98,6 +73,10 @@ impl From for NonceCounterSealError { /// Monotonic deterministic nonce generator for AES-GCM. /// +/// Each 96-bit nonce consists of a caller-selected 32-bit fixed prefix and a +/// 64-bit big-endian invocation counter, following the deterministic IV shape +/// from SP 800-38D. +/// /// The counter is intentionally not `Clone` or `Copy`. One instance owns one /// nonce stream. If you need restart-safe continuation, persist /// [`next_counter`](Self::next_counter) and restore with @@ -108,6 +87,15 @@ pub struct NonceCounter { _cipher: PhantomData Cipher>, } +impl fmt::Debug for NonceCounter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("NonceCounter") + .field("fixed_prefix", &self.fixed_prefix) + .field("next_counter", &self.next) + .finish() + } +} + impl NonceCounter { /// Fixed per-stream prefix length in bytes. pub const FIXED_PREFIX_LEN: usize = FIXED_PREFIX_LEN; @@ -355,22 +343,41 @@ impl NonceCounter { #[cfg(test)] mod tests { + use alloc::format; + use core::marker::PhantomData; + use super::{Aes128Gcm, Aes256Gcm, NonceCounter, NonceCounterSealError}; use crate::{ Aes128GcmKey, Aes256GcmKey, aead::{Nonce96, SealError}, }; + #[test] + fn nonce_counter_debug_does_not_require_cipher_debug() { + struct CipherWithoutDebug; + + let counter = NonceCounter:: { + fixed_prefix: *b"test", + next: 7, + _cipher: PhantomData, + }; + + assert_eq!( + format!("{counter:?}"), + "NonceCounter { fixed_prefix: [116, 101, 115, 116], next_counter: 7 }" + ); + } + #[test] fn aes_gcm_nonce_counter_formats_prefix_and_counter() { let mut counter = NonceCounter::::new(*b"conn"); assert_eq!( - counter.next_nonce().unwrap(), + counter.next_nonce().expect("first nonce should be available"), Nonce96::from_bytes([b'c', b'o', b'n', b'n', 0, 0, 0, 0, 0, 0, 0, 0]) ); assert_eq!( - counter.next_nonce().unwrap(), + counter.next_nonce().expect("second nonce should be available"), Nonce96::from_bytes([b'c', b'o', b'n', b'n', 0, 0, 0, 0, 0, 0, 0, 1]) ); assert_eq!(counter.issued(), 2); @@ -382,10 +389,14 @@ mod tests { let mut counter = NonceCounter::::new(*b"sess"); let mut sealed = [0u8; 4 + Aes256Gcm::TAG_SIZE]; - let nonce = counter.encrypt(&cipher, b"hdr", b"data", &mut sealed).unwrap(); + let nonce = counter + .encrypt(&cipher, b"hdr", b"data", &mut sealed) + .expect("valid inputs should seal"); let mut opened = [0u8; 4]; - cipher.decrypt(&nonce, b"hdr", &sealed, &mut opened).unwrap(); + cipher + .decrypt(&nonce, b"hdr", &sealed, &mut opened) + .expect("freshly sealed ciphertext should open"); assert_eq!(&opened, b"data"); assert_eq!(counter.next_counter(), 1); } @@ -396,7 +407,9 @@ mod tests { let mut counter = NonceCounter::::new(*b"bufr"); let mut out = [0u8; 3]; - let err = counter.encrypt(&cipher, b"", b"data", &mut out).unwrap_err(); + let err = counter + .encrypt(&cipher, b"", b"data", &mut out) + .expect_err("undersized output should be rejected"); assert_eq!(err, NonceCounterSealError::from(SealError::buffer())); assert_eq!(counter.next_counter(), 1); } @@ -404,11 +417,14 @@ mod tests { #[test] fn aes_gcm_nonce_counter_exhausts_cleanly() { let mut counter = - NonceCounter::::with_counter(*b"last", NonceCounter::::MAX_MESSAGES.strict_sub(1)).unwrap(); + NonceCounter::::with_counter(*b"last", NonceCounter::::MAX_MESSAGES.strict_sub(1)) + .expect("last permitted counter should be accepted"); - assert!(counter.next_nonce().is_ok()); + counter.next_nonce().expect("last nonce should be available"); assert_eq!(counter.remaining(), 0); - assert!(counter.next_nonce().is_err()); + counter + .next_nonce() + .expect_err("exhausted counter should reject another nonce"); } #[test] @@ -417,13 +433,15 @@ mod tests { // greater than or equal to MAX_MESSAGES. Pin both the equality and the // strictly-greater branch so a future relaxation cannot silently widen // the deterministic-IV budget past the SP 800-38D limit. - assert!(NonceCounter::::with_counter(*b"oflw", NonceCounter::::MAX_MESSAGES).is_err()); - assert!(NonceCounter::::with_counter(*b"oflw", u64::MAX).is_err()); + NonceCounter::::with_counter(*b"oflw", NonceCounter::::MAX_MESSAGES) + .expect_err("maximum counter should be rejected"); + NonceCounter::::with_counter(*b"oflw", u64::MAX).expect_err("out-of-range counter should be rejected"); } #[test] fn aes_gcm_nonce_counters_report_resumed_state() { - let counter256 = NonceCounter::::with_counter(*b"r256", 37).unwrap(); + let counter256 = + NonceCounter::::with_counter(*b"r256", 37).expect("in-range counter should be accepted"); assert_eq!(counter256.fixed_prefix(), *b"r256"); assert_eq!(counter256.next_counter(), 37); assert_eq!(counter256.issued(), 37); @@ -432,7 +450,8 @@ mod tests { NonceCounter::::MAX_MESSAGES.strict_sub(37) ); - let counter128 = NonceCounter::::with_counter(*b"r128", 73).unwrap(); + let counter128 = + NonceCounter::::with_counter(*b"r128", 73).expect("in-range counter should be accepted"); assert_eq!(counter128.fixed_prefix(), *b"r128"); assert_eq!(counter128.next_counter(), 73); assert_eq!(counter128.issued(), 73); @@ -448,10 +467,14 @@ mod tests { let mut counter = NonceCounter::::new(*b"sess"); let mut sealed = [0u8; 4 + Aes128Gcm::TAG_SIZE]; - let nonce = counter.encrypt(&cipher, b"hdr", b"data", &mut sealed).unwrap(); + let nonce = counter + .encrypt(&cipher, b"hdr", b"data", &mut sealed) + .expect("valid inputs should seal"); let mut opened = [0u8; 4]; - cipher.decrypt(&nonce, b"hdr", &sealed, &mut opened).unwrap(); + cipher + .decrypt(&nonce, b"hdr", &sealed, &mut opened) + .expect("freshly sealed ciphertext should open"); assert_eq!(&opened, b"data"); assert_eq!(counter.next_counter(), 1); } @@ -461,11 +484,11 @@ mod tests { let mut counter = NonceCounter::::new(*b"conn"); assert_eq!( - counter.next_nonce().unwrap(), + counter.next_nonce().expect("first nonce should be available"), Nonce96::from_bytes([b'c', b'o', b'n', b'n', 0, 0, 0, 0, 0, 0, 0, 0]) ); assert_eq!( - counter.next_nonce().unwrap(), + counter.next_nonce().expect("second nonce should be available"), Nonce96::from_bytes([b'c', b'o', b'n', b'n', 0, 0, 0, 0, 0, 0, 0, 1]) ); assert_eq!(counter.issued(), 2); @@ -477,7 +500,9 @@ mod tests { let mut counter = NonceCounter::::new(*b"bufr"); let mut out = [0u8; 3]; - let err = counter.encrypt(&cipher, b"", b"data", &mut out).unwrap_err(); + let err = counter + .encrypt(&cipher, b"", b"data", &mut out) + .expect_err("undersized output should be rejected"); assert_eq!(err, NonceCounterSealError::from(SealError::buffer())); assert_eq!(counter.next_counter(), 1); } @@ -485,11 +510,14 @@ mod tests { #[test] fn aes128_gcm_nonce_counter_exhausts_cleanly() { let mut counter = - NonceCounter::::with_counter(*b"last", NonceCounter::::MAX_MESSAGES.strict_sub(1)).unwrap(); + NonceCounter::::with_counter(*b"last", NonceCounter::::MAX_MESSAGES.strict_sub(1)) + .expect("last permitted counter should be accepted"); - assert!(counter.next_nonce().is_ok()); + counter.next_nonce().expect("last nonce should be available"); assert_eq!(counter.remaining(), 0); - assert!(counter.next_nonce().is_err()); + counter + .next_nonce() + .expect_err("exhausted counter should reject another nonce"); } #[test] @@ -497,7 +525,8 @@ mod tests { // Pin both the equality and strictly-greater branch so a future // relaxation cannot silently widen the deterministic-IV budget past // the SP 800-38D limit. - assert!(NonceCounter::::with_counter(*b"oflw", NonceCounter::::MAX_MESSAGES).is_err()); - assert!(NonceCounter::::with_counter(*b"oflw", u64::MAX).is_err()); + NonceCounter::::with_counter(*b"oflw", NonceCounter::::MAX_MESSAGES) + .expect_err("maximum counter should be rejected"); + NonceCounter::::with_counter(*b"oflw", u64::MAX).expect_err("out-of-range counter should be rejected"); } } diff --git a/src/aead/poly1305.rs b/src/aead/poly1305.rs index 557d52f1..08e0345b 100644 --- a/src/aead/poly1305.rs +++ b/src/aead/poly1305.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! Portable Poly1305 core. #[cfg(feature = "std")] @@ -21,8 +19,10 @@ const RISCV64_PAR4_MIN: u64 = 4096; type ComputeBlockFn = fn(&mut State, &[u8; 16], bool); #[cfg(feature = "std")] +#[cfg(feature = "xchacha20poly1305")] static XCHACHA20POLY1305_COMPUTE_BLOCK_DISPATCH: OnceCache = OnceCache::new(); #[cfg(feature = "std")] +#[cfg(feature = "chacha20poly1305")] static CHACHA20POLY1305_COMPUTE_BLOCK_DISPATCH: OnceCache = OnceCache::new(); #[inline] @@ -32,6 +32,29 @@ fn load_u32_le(input: &[u8]) -> u32 { u32::from_le_bytes(bytes) } +#[cfg(any( + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "wasm32", +))] +#[inline(always)] +fn low_u32(value: u64) -> u32 { + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) +} + +#[cfg(any( + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "wasm32", +))] +#[inline(always)] +fn add_limb_product(accumulator: u64, left: u32, right: u32) -> u64 { + accumulator.wrapping_add(u64::from(left).wrapping_mul(u64::from(right))) +} + #[cfg(any( all(target_arch = "powerpc64", target_endian = "little"), target_arch = "riscv64", @@ -52,10 +75,10 @@ fn compute_block_scalar_reduction( let r3 = state.r[3]; let r4 = state.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = r1.wrapping_mul(5); + let s2 = r2.wrapping_mul(5); + let s3 = r3.wrapping_mul(5); + let s4 = r4.wrapping_mul(5); let mut h0 = state.h[0]; let mut h1 = state.h[1]; @@ -69,31 +92,31 @@ fn compute_block_scalar_reduction( h3 = h3.wrapping_add((load_u32_le(&block[9..13]) >> 6) & LIMB_MASK); h4 = h4.wrapping_add((load_u32_le(&block[12..16]) >> 8) | hibit); - let d0 = sum4_mul([h0, h1, h2, h3], [r0, s4, s3, s2]) + (u64::from(h4) * u64::from(s1)); - let mut d1 = sum4_mul([h0, h1, h2, h3], [r1, r0, s4, s3]) + (u64::from(h4) * u64::from(s2)); - let mut d2 = sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4]) + (u64::from(h4) * u64::from(s3)); - let mut d3 = sum4_mul([h0, h1, h2, h3], [r3, r2, r1, r0]) + (u64::from(h4) * u64::from(s4)); - let mut d4 = sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]) + (u64::from(h4) * u64::from(r0)); + let d0 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r0, s4, s3, s2]), h4, s1); + let mut d1 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r1, r0, s4, s3]), h4, s2); + let mut d2 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4]), h4, s3); + let mut d3 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r3, r2, r1, r0]), h4, s4); + let mut d4 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]), h4, r0); - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); + let mut c = low_u32(d0 >> 26); + h0 = low_u32(d0) & LIMB_MASK; + d1 = d1.wrapping_add(u64::from(c)); - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); + c = low_u32(d1 >> 26); + h1 = low_u32(d1) & LIMB_MASK; + d2 = d2.wrapping_add(u64::from(c)); - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); + c = low_u32(d2 >> 26); + h2 = low_u32(d2) & LIMB_MASK; + d3 = d3.wrapping_add(u64::from(c)); - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); + c = low_u32(d3 >> 26); + h3 = low_u32(d3) & LIMB_MASK; + d4 = d4.wrapping_add(u64::from(c)); - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + c = low_u32(d4 >> 26); + h4 = low_u32(d4) & LIMB_MASK; + h0 = h0.wrapping_add(c.wrapping_mul(5)); c = h0 >> 26; h0 &= LIMB_MASK; @@ -120,12 +143,15 @@ fn compute_block_resolved(primitive: AeadPrimitive) -> ComputeBlockFn { #[cfg(feature = "std")] { match primitive { + #[cfg(feature = "xchacha20poly1305")] AeadPrimitive::XChaCha20Poly1305 => { XCHACHA20POLY1305_COMPUTE_BLOCK_DISPATCH.get_or_init(|| resolve_compute_block(primitive)) } + #[cfg(feature = "chacha20poly1305")] AeadPrimitive::ChaCha20Poly1305 => { CHACHA20POLY1305_COMPUTE_BLOCK_DISPATCH.get_or_init(|| resolve_compute_block(primitive)) } + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] _ => resolve_compute_block(primitive), } } @@ -159,21 +185,84 @@ fn resolve_compute_block(primitive: AeadPrimitive) -> ComputeBlockFn { #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2")] +/// Absorbs one Poly1305 block with the x86-64 AVX2 multiplier. +/// +/// # Safety +/// +/// The current CPU must support AVX2. `state` must satisfy the internal clamped-key and reduced-accumulator limb +/// bounds established by `State::new` and the Poly1305 block kernels. unsafe fn compute_block_x86_avx2(state: &mut State, block: &[u8; 16], partial: bool) { use core::arch::x86_64::{__m256i, _mm256_mul_epu32, _mm256_setr_epi32, _mm256_storeu_si256}; + use core::ptr::NonNull; #[inline(always)] fn sum4_mul(lhs: [u32; 4], rhs: [u32; 4]) -> u64 { - // SAFETY: the enclosing kernel enables AVX2 and the destination array is a valid - // unaligned store target for one `__m256i`. + let mut lanes = [0u64; 4]; + let destination = NonNull::from(&mut lanes).cast::<__m256i>().as_ptr(); + + // SAFETY: this helper is called only from the enclosing AVX2 kernel. `destination` retains the provenance of + // the writable 32-byte `lanes` array, and `_mm256_storeu_si256` permits its 8-byte alignment. unsafe { - let a = _mm256_setr_epi32(lhs[0] as i32, 0, lhs[1] as i32, 0, lhs[2] as i32, 0, lhs[3] as i32, 0); - let b = _mm256_setr_epi32(rhs[0] as i32, 0, rhs[1] as i32, 0, rhs[2] as i32, 0, rhs[3] as i32, 0); + let a = _mm256_setr_epi32( + lhs[0].cast_signed(), + 0, + lhs[1].cast_signed(), + 0, + lhs[2].cast_signed(), + 0, + lhs[3].cast_signed(), + 0, + ); + let b = _mm256_setr_epi32( + rhs[0].cast_signed(), + 0, + rhs[1].cast_signed(), + 0, + rhs[2].cast_signed(), + 0, + rhs[3].cast_signed(), + 0, + ); let products = _mm256_mul_epu32(a, b); - let mut lanes = [0u64; 4]; - _mm256_storeu_si256(lanes.as_mut_ptr() as *mut __m256i, products); - lanes[0] + lanes[1] + lanes[2] + lanes[3] + _mm256_storeu_si256(destination, products); } + + let sum = u128::from(lanes[0]) + .strict_add(u128::from(lanes[1])) + .strict_add(u128::from(lanes[2])) + .strict_add(u128::from(lanes[3])); + debug_assert!(sum <= u128::from(u64::MAX)); + let [b0, b1, b2, b3, b4, b5, b6, b7, _, _, _, _, _, _, _, _] = sum.to_le_bytes(); + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]) + } + + #[inline(always)] + fn fivefold_limb(limb: u32) -> u32 { + const MAX_UNSCALED: u32 = 858_993_459; + debug_assert!(limb <= MAX_UNSCALED); + + let product = u64::from(limb).strict_mul(5); + let [b0, b1, b2, b3, _, _, _, _] = product.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + + #[inline(always)] + fn sum5_mul(lhs: [u32; 5], rhs: [u32; 5]) -> u64 { + let [l0, l1, l2, l3, l4] = lhs; + let [r0, r1, r2, r3, r4] = rhs; + let sum = + u128::from(sum4_mul([l0, l1, l2, l3], [r0, r1, r2, r3])).strict_add(u128::from(l4).strict_mul(u128::from(r4))); + debug_assert!(sum <= u128::from(u64::MAX)); + + let [b0, b1, b2, b3, b4, b5, b6, b7, _, _, _, _, _, _, _, _] = sum.to_le_bytes(); + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]) + } + + #[inline(always)] + fn narrow_limb(value: u64) -> u32 { + debug_assert_eq!(value >> u32::BITS, 0); + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) } let hibit = if partial { 0 } else { FULL_BLOCK_HIBIT }; @@ -184,10 +273,10 @@ unsafe fn compute_block_x86_avx2(state: &mut State, block: &[u8; 16], partial: b let r3 = state.r[3]; let r4 = state.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = fivefold_limb(r1); + let s2 = fivefold_limb(r2); + let s3 = fivefold_limb(r3); + let s4 = fivefold_limb(r4); let mut h0 = state.h[0]; let mut h1 = state.h[1]; @@ -201,33 +290,33 @@ unsafe fn compute_block_x86_avx2(state: &mut State, block: &[u8; 16], partial: b h3 = h3.wrapping_add((load_u32_le(&block[9..13]) >> 6) & LIMB_MASK); h4 = h4.wrapping_add((load_u32_le(&block[12..16]) >> 8) | hibit); - let d0 = sum4_mul([h0, h1, h2, h3], [r0, s4, s3, s2]) + (u64::from(h4) * u64::from(s1)); - let mut d1 = sum4_mul([h0, h1, h2, h3], [r1, r0, s4, s3]) + (u64::from(h4) * u64::from(s2)); - let mut d2 = sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4]) + (u64::from(h4) * u64::from(s3)); - let mut d3 = sum4_mul([h0, h1, h2, h3], [r3, r2, r1, r0]) + (u64::from(h4) * u64::from(s4)); - let mut d4 = sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]) + (u64::from(h4) * u64::from(r0)); + let d0 = sum5_mul([h0, h1, h2, h3, h4], [r0, s4, s3, s2, s1]); + let mut d1 = sum5_mul([h0, h1, h2, h3, h4], [r1, r0, s4, s3, s2]); + let mut d2 = sum5_mul([h0, h1, h2, h3, h4], [r2, r1, r0, s4, s3]); + let mut d3 = sum5_mul([h0, h1, h2, h3, h4], [r3, r2, r1, r0, s4]); + let mut d4 = sum5_mul([h0, h1, h2, h3, h4], [r4, r3, r2, r1, r0]); - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); + let mut c = d0 >> 26; + h0 = narrow_limb(d0 & u64::from(LIMB_MASK)); + d1 = d1.strict_add(c); - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); + c = d1 >> 26; + h1 = narrow_limb(d1 & u64::from(LIMB_MASK)); + d2 = d2.strict_add(c); - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); + c = d2 >> 26; + h2 = narrow_limb(d2 & u64::from(LIMB_MASK)); + d3 = d3.strict_add(c); - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); + c = d3 >> 26; + h3 = narrow_limb(d3 & u64::from(LIMB_MASK)); + d4 = d4.strict_add(c); - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + c = d4 >> 26; + h4 = narrow_limb(d4 & u64::from(LIMB_MASK)); + h0 = h0.wrapping_add(fivefold_limb(narrow_limb(c))); - c = h0 >> 26; + let c = h0 >> 26; h0 &= LIMB_MASK; h1 = h1.wrapping_add(c); @@ -236,58 +325,90 @@ unsafe fn compute_block_x86_avx2(state: &mut State, block: &[u8; 16], partial: b #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq")] +/// Absorbs one Poly1305 block with the x86-64 AVX-512 multiplier. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, and AVX-512DQ. `state` must satisfy the internal +/// clamped-key and reduced-accumulator limb bounds established by `State::new` and the Poly1305 block kernels. unsafe fn compute_block_x86_avx512(state: &mut State, block: &[u8; 16], partial: bool) { use core::arch::x86_64::{__m512i, _mm512_mul_epu32, _mm512_setr_epi32, _mm512_storeu_si512}; + use core::ptr::NonNull; + + #[inline(always)] + fn narrow_sum(value: u128) -> u64 { + debug_assert!(value <= u128::from(u64::MAX)); + let [b0, b1, b2, b3, b4, b5, b6, b7, _, _, _, _, _, _, _, _] = value.to_le_bytes(); + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]) + } + + #[inline(always)] + fn fivefold_limb(limb: u32) -> u32 { + const MAX_UNSCALED: u32 = 858_993_459; + debug_assert!(limb <= MAX_UNSCALED); + + let product = u64::from(limb).strict_mul(5); + let [b0, b1, b2, b3, _, _, _, _] = product.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } #[inline(always)] fn pair_sum4_mul(lhs: [u32; 4], rhs_lo: [u32; 4], rhs_hi: [u32; 4]) -> (u64, u64) { - // SAFETY: the enclosing kernel enables AVX-512F, and the destination array is a valid - // unaligned store target for one `__m512i`. + let mut lanes = [0u64; 8]; + let destination = NonNull::from(&mut lanes).cast::<__m512i>().as_ptr(); + + // SAFETY: this helper is called only from the enclosing AVX-512 kernel. `destination` retains the provenance of + // the writable 64-byte `lanes` array, and `_mm512_storeu_si512` permits its 8-byte alignment. unsafe { let a = _mm512_setr_epi32( - lhs[0] as i32, + lhs[0].cast_signed(), 0, - lhs[1] as i32, + lhs[1].cast_signed(), 0, - lhs[2] as i32, + lhs[2].cast_signed(), 0, - lhs[3] as i32, + lhs[3].cast_signed(), 0, - lhs[0] as i32, + lhs[0].cast_signed(), 0, - lhs[1] as i32, + lhs[1].cast_signed(), 0, - lhs[2] as i32, + lhs[2].cast_signed(), 0, - lhs[3] as i32, + lhs[3].cast_signed(), 0, ); let b = _mm512_setr_epi32( - rhs_lo[0] as i32, + rhs_lo[0].cast_signed(), 0, - rhs_lo[1] as i32, + rhs_lo[1].cast_signed(), 0, - rhs_lo[2] as i32, + rhs_lo[2].cast_signed(), 0, - rhs_lo[3] as i32, + rhs_lo[3].cast_signed(), 0, - rhs_hi[0] as i32, + rhs_hi[0].cast_signed(), 0, - rhs_hi[1] as i32, + rhs_hi[1].cast_signed(), 0, - rhs_hi[2] as i32, + rhs_hi[2].cast_signed(), 0, - rhs_hi[3] as i32, + rhs_hi[3].cast_signed(), 0, ); let products = _mm512_mul_epu32(a, b); - let mut lanes = [0u64; 8]; - _mm512_storeu_si512(lanes.as_mut_ptr() as *mut __m512i, products); - ( - lanes[0] + lanes[1] + lanes[2] + lanes[3], - lanes[4] + lanes[5] + lanes[6] + lanes[7], - ) + _mm512_storeu_si512(destination, products); } + + let low = u128::from(lanes[0]) + .strict_add(u128::from(lanes[1])) + .strict_add(u128::from(lanes[2])) + .strict_add(u128::from(lanes[3])); + let high = u128::from(lanes[4]) + .strict_add(u128::from(lanes[5])) + .strict_add(u128::from(lanes[6])) + .strict_add(u128::from(lanes[7])); + (narrow_sum(low), narrow_sum(high)) } #[inline(always)] @@ -295,6 +416,19 @@ unsafe fn compute_block_x86_avx512(state: &mut State, block: &[u8; 16], partial: pair_sum4_mul(lhs, rhs, [0; 4]).0 } + #[inline(always)] + fn add_product(base: u64, lhs: u32, rhs: u32) -> u64 { + let sum = u128::from(base).strict_add(u128::from(lhs).strict_mul(u128::from(rhs))); + narrow_sum(sum) + } + + #[inline(always)] + fn narrow_limb(value: u64) -> u32 { + debug_assert_eq!(value >> u32::BITS, 0); + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + let hibit = if partial { 0 } else { FULL_BLOCK_HIBIT }; let r0 = state.r[0]; @@ -303,10 +437,10 @@ unsafe fn compute_block_x86_avx512(state: &mut State, block: &[u8; 16], partial: let r3 = state.r[3]; let r4 = state.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = fivefold_limb(r1); + let s2 = fivefold_limb(r2); + let s3 = fivefold_limb(r3); + let s4 = fivefold_limb(r4); let mut h0 = state.h[0]; let mut h1 = state.h[1]; @@ -324,39 +458,44 @@ unsafe fn compute_block_x86_avx512(state: &mut State, block: &[u8; 16], partial: let (d2_base, d3_base) = pair_sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4], [r3, r2, r1, r0]); let d4_base = single_sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]); - let d0 = d0_base + (u64::from(h4) * u64::from(s1)); - let mut d1 = d1_base + (u64::from(h4) * u64::from(s2)); - let mut d2 = d2_base + (u64::from(h4) * u64::from(s3)); - let mut d3 = d3_base + (u64::from(h4) * u64::from(s4)); - let mut d4 = d4_base + (u64::from(h4) * u64::from(r0)); + let d0 = add_product(d0_base, h4, s1); + let mut d1 = add_product(d1_base, h4, s2); + let mut d2 = add_product(d2_base, h4, s3); + let mut d3 = add_product(d3_base, h4, s4); + let mut d4 = add_product(d4_base, h4, r0); - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); + let mut c = d0 >> 26; + h0 = narrow_limb(d0 & u64::from(LIMB_MASK)); + d1 = d1.strict_add(c); - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); + c = d1 >> 26; + h1 = narrow_limb(d1 & u64::from(LIMB_MASK)); + d2 = d2.strict_add(c); - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); + c = d2 >> 26; + h2 = narrow_limb(d2 & u64::from(LIMB_MASK)); + d3 = d3.strict_add(c); - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); + c = d3 >> 26; + h3 = narrow_limb(d3 & u64::from(LIMB_MASK)); + d4 = d4.strict_add(c); - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + c = d4 >> 26; + h4 = narrow_limb(d4 & u64::from(LIMB_MASK)); + h0 = h0.wrapping_add(fivefold_limb(narrow_limb(c))); - c = h0 >> 26; + let c = h0 >> 26; h0 &= LIMB_MASK; h1 = h1.wrapping_add(c); state.h = [h0, h1, h2, h3, h4]; } +/// Absorbs one full Poly1305 block with the AArch64 NEON multiplier. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] unsafe fn compute_block_aarch64_neon(state: &mut State, block: &[u8; 16], partial: bool) { @@ -379,6 +518,35 @@ unsafe fn compute_block_aarch64_neon(state: &mut State, block: &[u8; 16], partia } } + #[inline(always)] + fn fivefold_limb(limb: u32) -> u32 { + const MAX_UNSCALED: u32 = 858_993_459; + debug_assert!(limb <= MAX_UNSCALED); + + let product = u64::from(limb).strict_mul(5); + let [b0, b1, b2, b3, _, _, _, _] = product.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + + #[inline(always)] + fn sum5_mul(lhs: [u32; 5], rhs: [u32; 5]) -> u64 { + let [l0, l1, l2, l3, l4] = lhs; + let [r0, r1, r2, r3, r4] = rhs; + let sum = + u128::from(sum4_mul([l0, l1, l2, l3], [r0, r1, r2, r3])).strict_add(u128::from(l4).strict_mul(u128::from(r4))); + debug_assert!(sum <= u128::from(u64::MAX)); + + let [b0, b1, b2, b3, b4, b5, b6, b7, _, _, _, _, _, _, _, _] = sum.to_le_bytes(); + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]) + } + + #[inline(always)] + fn narrow_limb(value: u64) -> u32 { + debug_assert_eq!(value >> u32::BITS, 0); + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + let hibit = if partial { 0 } else { FULL_BLOCK_HIBIT }; let r0 = state.r[0]; @@ -387,10 +555,10 @@ unsafe fn compute_block_aarch64_neon(state: &mut State, block: &[u8; 16], partia let r3 = state.r[3]; let r4 = state.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = fivefold_limb(r1); + let s2 = fivefold_limb(r2); + let s3 = fivefold_limb(r3); + let s4 = fivefold_limb(r4); let mut h0 = state.h[0]; let mut h1 = state.h[1]; @@ -404,33 +572,33 @@ unsafe fn compute_block_aarch64_neon(state: &mut State, block: &[u8; 16], partia h3 = h3.wrapping_add((load_u32_le(&block[9..13]) >> 6) & LIMB_MASK); h4 = h4.wrapping_add((load_u32_le(&block[12..16]) >> 8) | hibit); - let d0 = sum4_mul([h0, h1, h2, h3], [r0, s4, s3, s2]) + (u64::from(h4) * u64::from(s1)); - let mut d1 = sum4_mul([h0, h1, h2, h3], [r1, r0, s4, s3]) + (u64::from(h4) * u64::from(s2)); - let mut d2 = sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4]) + (u64::from(h4) * u64::from(s3)); - let mut d3 = sum4_mul([h0, h1, h2, h3], [r3, r2, r1, r0]) + (u64::from(h4) * u64::from(s4)); - let mut d4 = sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]) + (u64::from(h4) * u64::from(r0)); + let d0 = sum5_mul([h0, h1, h2, h3, h4], [r0, s4, s3, s2, s1]); + let mut d1 = sum5_mul([h0, h1, h2, h3, h4], [r1, r0, s4, s3, s2]); + let mut d2 = sum5_mul([h0, h1, h2, h3, h4], [r2, r1, r0, s4, s3]); + let mut d3 = sum5_mul([h0, h1, h2, h3, h4], [r3, r2, r1, r0, s4]); + let mut d4 = sum5_mul([h0, h1, h2, h3, h4], [r4, r3, r2, r1, r0]); - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); + let mut c = d0 >> 26; + h0 = narrow_limb(d0 & u64::from(LIMB_MASK)); + d1 = d1.strict_add(c); - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); + c = d1 >> 26; + h1 = narrow_limb(d1 & u64::from(LIMB_MASK)); + d2 = d2.strict_add(c); - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); + c = d2 >> 26; + h2 = narrow_limb(d2 & u64::from(LIMB_MASK)); + d3 = d3.strict_add(c); - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); + c = d3 >> 26; + h3 = narrow_limb(d3 & u64::from(LIMB_MASK)); + d4 = d4.strict_add(c); - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + c = d4 >> 26; + h4 = narrow_limb(d4 & u64::from(LIMB_MASK)); + h0 = h0.wrapping_add(fivefold_limb(narrow_limb(c))); - c = h0 >> 26; + let c = h0 >> 26; h0 &= LIMB_MASK; h1 = h1.wrapping_add(c); @@ -439,6 +607,12 @@ unsafe fn compute_block_aarch64_neon(state: &mut State, block: &[u8; 16], partia #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] +/// Absorbs one Poly1305 block with the WASM SIMD128 multiplier. +/// +/// # Safety +/// +/// The current WebAssembly instance must support SIMD128. `state` must satisfy the internal clamped-key and +/// reduced-accumulator limb bounds established by `State::new` and the Poly1305 block kernels. unsafe fn compute_block_wasm_simd128(state: &mut State, block: &[u8; 16], partial: bool) { use core::arch::wasm32::{i64x2_add, u32x4, u64x2_extmul_high_u32x4, u64x2_extmul_low_u32x4, u64x2_extract_lane}; @@ -449,7 +623,7 @@ unsafe fn compute_block_wasm_simd128(state: &mut State, block: &[u8; 16], partia let lo = u64x2_extmul_low_u32x4(a, b); let hi = u64x2_extmul_high_u32x4(a, b); let sum = i64x2_add(lo, hi); - u64x2_extract_lane::<0>(sum) + u64x2_extract_lane::<1>(sum) + u64x2_extract_lane::<0>(sum).wrapping_add(u64x2_extract_lane::<1>(sum)) } let hibit = if partial { 0 } else { FULL_BLOCK_HIBIT }; @@ -460,10 +634,10 @@ unsafe fn compute_block_wasm_simd128(state: &mut State, block: &[u8; 16], partia let r3 = state.r[3]; let r4 = state.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = r1.wrapping_mul(5); + let s2 = r2.wrapping_mul(5); + let s3 = r3.wrapping_mul(5); + let s4 = r4.wrapping_mul(5); let mut h0 = state.h[0]; let mut h1 = state.h[1]; @@ -477,31 +651,31 @@ unsafe fn compute_block_wasm_simd128(state: &mut State, block: &[u8; 16], partia h3 = h3.wrapping_add((load_u32_le(&block[9..13]) >> 6) & LIMB_MASK); h4 = h4.wrapping_add((load_u32_le(&block[12..16]) >> 8) | hibit); - let d0 = sum4_mul([h0, h1, h2, h3], [r0, s4, s3, s2]) + (u64::from(h4) * u64::from(s1)); - let mut d1 = sum4_mul([h0, h1, h2, h3], [r1, r0, s4, s3]) + (u64::from(h4) * u64::from(s2)); - let mut d2 = sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4]) + (u64::from(h4) * u64::from(s3)); - let mut d3 = sum4_mul([h0, h1, h2, h3], [r3, r2, r1, r0]) + (u64::from(h4) * u64::from(s4)); - let mut d4 = sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]) + (u64::from(h4) * u64::from(r0)); + let d0 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r0, s4, s3, s2]), h4, s1); + let mut d1 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r1, r0, s4, s3]), h4, s2); + let mut d2 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4]), h4, s3); + let mut d3 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r3, r2, r1, r0]), h4, s4); + let mut d4 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]), h4, r0); - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); + let mut c = low_u32(d0 >> 26); + h0 = low_u32(d0) & LIMB_MASK; + d1 = d1.wrapping_add(u64::from(c)); - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); + c = low_u32(d1 >> 26); + h1 = low_u32(d1) & LIMB_MASK; + d2 = d2.wrapping_add(u64::from(c)); - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); + c = low_u32(d2 >> 26); + h2 = low_u32(d2) & LIMB_MASK; + d3 = d3.wrapping_add(u64::from(c)); - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); + c = low_u32(d3 >> 26); + h3 = low_u32(d3) & LIMB_MASK; + d4 = d4.wrapping_add(u64::from(c)); - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + c = low_u32(d4 >> 26); + h4 = low_u32(d4) & LIMB_MASK; + h0 = h0.wrapping_add(c.wrapping_mul(5)); c = h0 >> 26; h0 &= LIMB_MASK; @@ -549,6 +723,39 @@ impl State { #[inline(always)] fn compute_block_portable(&mut self, block: &[u8; 16], partial: bool) { + #[inline(always)] + fn fivefold_limb(limb: u32) -> u32 { + const MAX_UNSCALED: u32 = 858_993_459; + debug_assert!(limb <= MAX_UNSCALED); + + let product = u64::from(limb).strict_mul(5); + let [b0, b1, b2, b3, _, _, _, _] = product.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + + #[inline(always)] + fn scalar_dot5(lhs: [u32; 5], rhs: [u32; 5]) -> u64 { + let [l0, l1, l2, l3, l4] = lhs; + let [r0, r1, r2, r3, r4] = rhs; + let sum = u128::from(l0) + .strict_mul(u128::from(r0)) + .strict_add(u128::from(l1).strict_mul(u128::from(r1))) + .strict_add(u128::from(l2).strict_mul(u128::from(r2))) + .strict_add(u128::from(l3).strict_mul(u128::from(r3))) + .strict_add(u128::from(l4).strict_mul(u128::from(r4))); + debug_assert!(sum <= u128::from(u64::MAX)); + + let [b0, b1, b2, b3, b4, b5, b6, b7, _, _, _, _, _, _, _, _] = sum.to_le_bytes(); + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]) + } + + #[inline(always)] + fn narrow_limb(value: u64) -> u32 { + debug_assert_eq!(value >> u32::BITS, 0); + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + let hibit = if partial { 0 } else { FULL_BLOCK_HIBIT }; let r0 = self.r[0]; @@ -557,10 +764,10 @@ impl State { let r3 = self.r[3]; let r4 = self.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = fivefold_limb(r1); + let s2 = fivefold_limb(r2); + let s3 = fivefold_limb(r3); + let s4 = fivefold_limb(r4); let mut h0 = self.h[0]; let mut h1 = self.h[1]; @@ -574,53 +781,33 @@ impl State { h3 = h3.wrapping_add((load_u32_le(&block[9..13]) >> 6) & LIMB_MASK); h4 = h4.wrapping_add((load_u32_le(&block[12..16]) >> 8) | hibit); - let d0 = (u64::from(h0) * u64::from(r0)) - + (u64::from(h1) * u64::from(s4)) - + (u64::from(h2) * u64::from(s3)) - + (u64::from(h3) * u64::from(s2)) - + (u64::from(h4) * u64::from(s1)); - let mut d1 = (u64::from(h0) * u64::from(r1)) - + (u64::from(h1) * u64::from(r0)) - + (u64::from(h2) * u64::from(s4)) - + (u64::from(h3) * u64::from(s3)) - + (u64::from(h4) * u64::from(s2)); - let mut d2 = (u64::from(h0) * u64::from(r2)) - + (u64::from(h1) * u64::from(r1)) - + (u64::from(h2) * u64::from(r0)) - + (u64::from(h3) * u64::from(s4)) - + (u64::from(h4) * u64::from(s3)); - let mut d3 = (u64::from(h0) * u64::from(r3)) - + (u64::from(h1) * u64::from(r2)) - + (u64::from(h2) * u64::from(r1)) - + (u64::from(h3) * u64::from(r0)) - + (u64::from(h4) * u64::from(s4)); - let mut d4 = (u64::from(h0) * u64::from(r4)) - + (u64::from(h1) * u64::from(r3)) - + (u64::from(h2) * u64::from(r2)) - + (u64::from(h3) * u64::from(r1)) - + (u64::from(h4) * u64::from(r0)); - - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); - - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); - - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); - - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); - - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + let d0 = scalar_dot5([h0, h1, h2, h3, h4], [r0, s4, s3, s2, s1]); + let mut d1 = scalar_dot5([h0, h1, h2, h3, h4], [r1, r0, s4, s3, s2]); + let mut d2 = scalar_dot5([h0, h1, h2, h3, h4], [r2, r1, r0, s4, s3]); + let mut d3 = scalar_dot5([h0, h1, h2, h3, h4], [r3, r2, r1, r0, s4]); + let mut d4 = scalar_dot5([h0, h1, h2, h3, h4], [r4, r3, r2, r1, r0]); - c = h0 >> 26; + let mut c = d0 >> 26; + h0 = narrow_limb(d0 & u64::from(LIMB_MASK)); + d1 = d1.strict_add(c); + + c = d1 >> 26; + h1 = narrow_limb(d1 & u64::from(LIMB_MASK)); + d2 = d2.strict_add(c); + + c = d2 >> 26; + h2 = narrow_limb(d2 & u64::from(LIMB_MASK)); + d3 = d3.strict_add(c); + + c = d3 >> 26; + h3 = narrow_limb(d3 & u64::from(LIMB_MASK)); + d4 = d4.strict_add(c); + + c = d4 >> 26; + h4 = narrow_limb(d4 & u64::from(LIMB_MASK)); + h0 = h0.wrapping_add(fivefold_limb(narrow_limb(c))); + + let c = h0 >> 26; h0 &= LIMB_MASK; h1 = h1.wrapping_add(c); @@ -629,14 +816,11 @@ impl State { #[cfg(test)] fn update_message(&mut self, message: &[u8], compute_block: ComputeBlockFn) { - let mut blocks = message.chunks_exact(16); - for chunk in &mut blocks { - let mut block = [0u8; 16]; - block.copy_from_slice(chunk); - compute_block(self, &block, false); + let (blocks, remainder) = message.as_chunks::<16>(); + for block in blocks { + compute_block(self, block, false); } - let remainder = blocks.remainder(); if remainder.is_empty() { return; } @@ -648,14 +832,11 @@ impl State { } fn update_padded_segment(&mut self, segment: &[u8], compute_block: ComputeBlockFn) { - let mut blocks = segment.chunks_exact(16); - for chunk in &mut blocks { - let mut block = [0u8; 16]; - block.copy_from_slice(chunk); - compute_block(self, &block, false); + let (blocks, remainder) = segment.as_chunks::<16>(); + for block in blocks { + compute_block(self, block, false); } - let remainder = blocks.remainder(); if remainder.is_empty() { return; } @@ -672,6 +853,22 @@ impl State { #[inline(always)] fn finalize_in_place(&mut self) -> [u8; 16] { + #[inline(always)] + fn fivefold_carry(carry: u32) -> u32 { + const MAX_UNSCALED: u32 = 858_993_459; + debug_assert!(carry <= MAX_UNSCALED); + + let product = u64::from(carry).strict_mul(5); + let [b0, b1, b2, b3, _, _, _, _] = product.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + + #[inline(always)] + fn low_word(value: u64) -> u32 { + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + let mut h0 = self.h[0]; let mut h1 = self.h[1]; let mut h2 = self.h[2]; @@ -692,7 +889,7 @@ impl State { c = h4 >> 26; h4 &= LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + h0 = h0.wrapping_add(fivefold_carry(c)); c = h0 >> 26; h0 &= LIMB_MASK; @@ -735,14 +932,14 @@ impl State { h2 = (h2 >> 12) | (h3 << 14); h3 = (h3 >> 18) | (h4 << 8); - let mut f = u64::from(h0) + u64::from(self.pad[0]); - h0 = f as u32; - f = u64::from(h1) + u64::from(self.pad[1]) + (f >> 32); - h1 = f as u32; - f = u64::from(h2) + u64::from(self.pad[2]) + (f >> 32); - h2 = f as u32; - f = u64::from(h3) + u64::from(self.pad[3]) + (f >> 32); - h3 = f as u32; + let mut f = u64::from(h0).strict_add(u64::from(self.pad[0])); + h0 = low_word(f); + f = u64::from(h1).strict_add(u64::from(self.pad[1])).strict_add(f >> 32); + h1 = low_word(f); + f = u64::from(h2).strict_add(u64::from(self.pad[2])).strict_add(f >> 32); + h2 = low_word(f); + f = u64::from(h3).strict_add(u64::from(self.pad[3])).strict_add(f >> 32); + h3 = low_word(f); let mut tag = [0u8; 16]; tag[0..4].copy_from_slice(&h0.to_le_bytes()); @@ -761,7 +958,6 @@ pub(crate) fn authenticate(message: &[u8], key: &[u8; 32]) -> [u8; 16] { state.finalize() } -#[cfg_attr(not(any(feature = "xchacha20poly1305", feature = "diag")), allow(dead_code))] pub(crate) fn authenticate_aead( primitive: AeadPrimitive, aad: &[u8], @@ -774,7 +970,8 @@ pub(crate) fn authenticate_aead( { use crate::platform::caps::x86; if lengths.total_at_least(64) && current_caps().has(x86::AVX2) { - return Ok(avx2_par4::authenticate_aead_par4(aad, ciphertext, key, lengths)); + // SAFETY: this branch verifies AVX2 before selecting the x86-64 parallel kernel. + return Ok(unsafe { avx2_par4::authenticate_aead_par4(aad, ciphertext, key, lengths) }); } } #[cfg(target_arch = "aarch64")] @@ -794,14 +991,11 @@ pub(crate) fn authenticate_aead( authenticate_aead_with(aad, ciphertext, key, compute_block_resolved(primitive), lengths) } -#[cfg_attr( - not(any( - test, - target_arch = "x86_64", - all(target_arch = "powerpc64", target_endian = "little") - )), - allow(dead_code) -)] +#[cfg(any( + test, + target_arch = "x86_64", + all(target_arch = "powerpc64", target_endian = "little") +))] fn authenticate_aead_portable_blocks( aad: &[u8], ciphertext: &[u8], @@ -820,19 +1014,16 @@ fn authenticate_aead_portable_blocks( tag } -#[cfg_attr( - not(any( - test, - target_arch = "x86_64", - all(target_arch = "powerpc64", target_endian = "little") - )), - allow(dead_code) -)] +#[cfg(any( + test, + target_arch = "x86_64", + all(target_arch = "powerpc64", target_endian = "little") +))] pub(crate) fn authenticate_aead_empty_text_portable(aad: &[u8], key: &[u8; 32]) -> [u8; 16] { authenticate_aead_portable_blocks(aad, &[], key, super::AeadByteLengths::from_usize(aad.len(), 0)) } -#[cfg_attr(not(all(target_arch = "powerpc64", target_endian = "little")), allow(dead_code))] +#[cfg(all(target_arch = "powerpc64", target_endian = "little"))] pub(crate) fn authenticate_aead_short_text_portable(aad: &[u8], ciphertext: &[u8], key: &[u8; 32]) -> [u8; 16] { authenticate_aead_portable_blocks( aad, @@ -843,13 +1034,23 @@ pub(crate) fn authenticate_aead_short_text_portable(aad: &[u8], ciphertext: &[u8 } #[cfg(feature = "diag")] +/// Computes a ChaCha20-Poly1305 authenticator through the selected Poly1305 backend. +/// +/// Returns `None` when the associated-data or ciphertext length cannot be encoded by the AEAD construction. pub fn diag_chacha20poly1305_authenticate_aead(aad: &[u8], ciphertext: &[u8], key: &[u8; 32]) -> Option<[u8; 16]> { - authenticate_aead(AeadPrimitive::ChaCha20Poly1305, aad, ciphertext, key).ok() + #[cfg(feature = "chacha20poly1305")] + let primitive = AeadPrimitive::ChaCha20Poly1305; + #[cfg(all(not(feature = "chacha20poly1305"), feature = "xchacha20poly1305"))] + let primitive = AeadPrimitive::XChaCha20Poly1305; + authenticate_aead(primitive, aad, ciphertext, key).ok() } #[cfg(feature = "diag")] #[unsafe(no_mangle)] #[inline(never)] +/// Computes a diagnostic Poly1305 tag after one block using the portable backend. +/// +/// `partial` suppresses the full-block high bit for a caller-prepared partial-block encoding. pub fn diag_poly1305_block_portable_digest(key: &[u8; 32], block: &[u8; 16], partial: bool) -> [u8; 16] { let mut state = State::new(key); state.compute_block_portable(block, partial); @@ -861,6 +1062,9 @@ pub fn diag_poly1305_block_portable_digest(key: &[u8; 32], block: &[u8; 16], par target_arch = "aarch64", any(target_os = "linux", target_os = "macos") ))] +/// Computes a ChaCha20-Poly1305 authenticator with the four-lane AArch64 NEON backend. +/// +/// Returns `None` when the associated-data or ciphertext length cannot be encoded by the AEAD construction. pub fn diag_chacha20poly1305_authenticate_aead_aarch64_neon_par4( aad: &[u8], ciphertext: &[u8], @@ -870,7 +1074,6 @@ pub fn diag_chacha20poly1305_authenticate_aead_aarch64_neon_par4( Some(aarch64_neon::authenticate_aead_par4(aad, ciphertext, key, lengths)) } -#[cfg_attr(not(any(feature = "xchacha20poly1305", feature = "diag", test)), allow(dead_code))] fn authenticate_aead_with( aad: &[u8], ciphertext: &[u8], @@ -894,7 +1097,6 @@ fn authenticate_aead_with( #[path = "poly1305/aarch64_neon.rs"] pub(crate) mod aarch64_neon; #[cfg(target_arch = "x86_64")] -#[allow(unsafe_op_in_unsafe_fn)] #[path = "poly1305/x86_64_avx2_par4.rs"] mod avx2_par4; #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] @@ -921,10 +1123,21 @@ mod tests { use super::authenticate; #[cfg(any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "riscv64"))] - use super::{ComputeBlockFn, State, authenticate_aead_with}; + use super::{ComputeBlockFn, authenticate_aead_with}; #[cfg(any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "riscv64"))] use crate::aead::AeadByteLengths; use crate::aead::targets::AeadPrimitive; + + fn primitive() -> AeadPrimitive { + #[cfg(feature = "chacha20poly1305")] + { + AeadPrimitive::ChaCha20Poly1305 + } + #[cfg(all(not(feature = "chacha20poly1305"), feature = "xchacha20poly1305"))] + { + AeadPrimitive::XChaCha20Poly1305 + } + } #[cfg(target_arch = "aarch64")] use crate::platform::caps::aarch64; #[cfg(target_arch = "riscv64")] @@ -932,6 +1145,15 @@ mod tests { #[cfg(target_arch = "x86_64")] use crate::platform::caps::x86; + fn patterned_bytes(length: usize, factor: usize, offset: usize) -> Vec { + (0..length) + .map(|index| { + let [byte, ..] = index.strict_mul(factor).strict_add(offset).to_le_bytes(); + byte + }) + .collect() + } + #[test] fn poly1305_matches_rfc_8439_section_2_5_2() { let key = [ @@ -965,10 +1187,8 @@ mod tests { 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, 0x06, 0x91, ]; - assert_eq!( - super::authenticate_aead(AeadPrimitive::ChaCha20Poly1305, &aad, &ciphertext, &poly_key).unwrap(), - expected - ); + let actual = super::authenticate_aead(primitive(), &aad, &ciphertext, &poly_key); + assert_eq!(actual, Ok(expected)); } #[test] @@ -976,13 +1196,12 @@ mod tests { let key = [0x5au8; 32]; for aad_len in [0usize, 1, 14, 15, 16, 17, 31, 32, 33, 63] { - let aad = (0..aad_len) - .map(|index| index.strict_mul(11).strict_add(7) as u8) - .collect::>(); - let expected = super::authenticate_aead(AeadPrimitive::ChaCha20Poly1305, &aad, &[], &key).unwrap(); + let aad = patterned_bytes(aad_len, 11, 7); + let expected = super::authenticate_aead(primitive(), &aad, &[], &key); let actual = super::authenticate_aead_empty_text_portable(&aad, &key); assert_eq!( - actual, expected, + expected, + Ok(actual), "empty-text authentication mismatch at aad_len={aad_len}" ); } @@ -990,8 +1209,8 @@ mod tests { #[cfg(any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "riscv64"))] fn authenticate_aead_portable(aad: &[u8], ciphertext: &[u8], key: &[u8; 32]) -> [u8; 16] { - let lengths = AeadByteLengths::try_new(aad.len(), ciphertext.len()).unwrap(); - authenticate_aead_with(aad, ciphertext, key, State::compute_block_portable, lengths).unwrap() + let lengths = AeadByteLengths::from_usize(aad.len(), ciphertext.len()); + super::authenticate_aead_portable_blocks(aad, ciphertext, key, lengths) } #[cfg(any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "riscv64"))] @@ -999,16 +1218,12 @@ mod tests { let key = [0x5au8; 32]; for aad_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 80] { for ciphertext_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 191, 256] { - let aad = (0..aad_len) - .map(|index| index.strict_mul(11).strict_add(7) as u8) - .collect::>(); - let ciphertext = (0..ciphertext_len) - .map(|index| index.strict_mul(17).strict_add(3) as u8) - .collect::>(); + let aad = patterned_bytes(aad_len, 11, 7); + let ciphertext = patterned_bytes(ciphertext_len, 17, 3); let portable = authenticate_aead_portable(&aad, &ciphertext, &key); - let lengths = AeadByteLengths::try_new(aad.len(), ciphertext.len()).unwrap(); - let accelerated = authenticate_aead_with(&aad, &ciphertext, &key, backend, lengths).unwrap(); - assert_eq!(accelerated, portable); + let lengths = AeadByteLengths::from_usize(aad.len(), ciphertext.len()); + let accelerated = authenticate_aead_with(&aad, &ciphertext, &key, backend, lengths); + assert_eq!(accelerated, Ok(portable)); } } } @@ -1063,11 +1278,12 @@ mod tests { let key = [0x5au8; 32]; for aad_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 48, 63, 64, 65, 80, 128] { for ct_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65, 191, 256, 1024, 4096] { - let aad: Vec = (0..aad_len).map(|i| i.strict_mul(11).strict_add(7) as u8).collect(); - let ct: Vec = (0..ct_len).map(|i| i.strict_mul(17).strict_add(3) as u8).collect(); - let lengths = AeadByteLengths::try_new(aad.len(), ct.len()).unwrap(); + let aad = patterned_bytes(aad_len, 11, 7); + let ct = patterned_bytes(ct_len, 17, 3); + let lengths = AeadByteLengths::from_usize(aad.len(), ct.len()); let portable = authenticate_aead_portable(&aad, &ct, &key); - let parallel = super::avx2_par4::authenticate_aead_par4(&aad, &ct, &key, lengths); + // SAFETY: the test returned above unless AVX2 is available. + let parallel = unsafe { super::avx2_par4::authenticate_aead_par4(&aad, &ct, &key, lengths) }; assert_eq!(parallel, portable, "mismatch at aad={aad_len} ct={ct_len}"); } } @@ -1083,9 +1299,9 @@ mod tests { let key = [0x5au8; 32]; for aad_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 48, 63, 64, 65, 80, 128] { for ct_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65, 191, 256, 1024, 4096] { - let aad: Vec = (0..aad_len).map(|i| i.strict_mul(11).strict_add(7) as u8).collect(); - let ct: Vec = (0..ct_len).map(|i| i.strict_mul(17).strict_add(3) as u8).collect(); - let lengths = AeadByteLengths::try_new(aad.len(), ct.len()).unwrap(); + let aad = patterned_bytes(aad_len, 11, 7); + let ct = patterned_bytes(ct_len, 17, 3); + let lengths = AeadByteLengths::from_usize(aad.len(), ct.len()); let portable = authenticate_aead_portable(&aad, &ct, &key); let parallel = super::aarch64_neon::authenticate_aead_par4(&aad, &ct, &key, lengths); assert_eq!(parallel, portable, "mismatch at aad={aad_len} ct={ct_len}"); @@ -1103,9 +1319,9 @@ mod tests { let key = [0x5au8; 32]; for aad_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 48, 63, 64, 65, 80, 128] { for ct_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65, 191, 256, 1024, 4096] { - let aad: Vec = (0..aad_len).map(|i| i.strict_mul(11).strict_add(7) as u8).collect(); - let ct: Vec = (0..ct_len).map(|i| i.strict_mul(17).strict_add(3) as u8).collect(); - let lengths = AeadByteLengths::try_new(aad.len(), ct.len()).unwrap(); + let aad = patterned_bytes(aad_len, 11, 7); + let ct = patterned_bytes(ct_len, 17, 3); + let lengths = AeadByteLengths::from_usize(aad.len(), ct.len()); let portable = authenticate_aead_portable(&aad, &ct, &key); let parallel = super::riscv64_vector::authenticate_aead_par4(&aad, &ct, &key, lengths); assert_eq!(parallel, portable, "mismatch at aad={aad_len} ct={ct_len}"); @@ -1113,6 +1329,24 @@ mod tests { } } + #[test] + #[cfg(target_arch = "x86_64")] + fn avx2_par4_handles_high_carry_reduction() { + if !crate::platform::caps().has(x86::AVX2) { + return; + } + + let key = [0xffu8; 32]; + let aad = [0xffu8; 257]; + let ct = [0xffu8; 4112]; + let lengths = AeadByteLengths::from_usize(aad.len(), ct.len()); + + let portable = authenticate_aead_portable(&aad, &ct, &key); + // SAFETY: the test returned above unless AVX2 is available. + let parallel = unsafe { super::avx2_par4::authenticate_aead_par4(&aad, &ct, &key, lengths) }; + assert_eq!(parallel, portable); + } + #[test] #[cfg(target_arch = "aarch64")] fn neon_par4_handles_high_carry_reduction() { @@ -1123,7 +1357,7 @@ mod tests { let key = [0xffu8; 32]; let aad = [0xffu8; 257]; let ct = [0xffu8; 4096]; - let lengths = AeadByteLengths::try_new(aad.len(), ct.len()).unwrap(); + let lengths = AeadByteLengths::from_usize(aad.len(), ct.len()); let portable = authenticate_aead_portable(&aad, &ct, &key); let parallel = super::aarch64_neon::authenticate_aead_par4(&aad, &ct, &key, lengths); @@ -1140,7 +1374,7 @@ mod tests { let key = [0xffu8; 32]; let aad = [0xffu8; 257]; let ct = [0xffu8; 4096]; - let lengths = AeadByteLengths::try_new(aad.len(), ct.len()).unwrap(); + let lengths = AeadByteLengths::from_usize(aad.len(), ct.len()); let portable = authenticate_aead_portable(&aad, &ct, &key); let parallel = super::riscv64_vector::authenticate_aead_par4(&aad, &ct, &key, lengths); @@ -1172,8 +1406,9 @@ mod tests { 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, 0x06, 0x91, ]; - let lengths = AeadByteLengths::try_new(aad.len(), ciphertext.len()).unwrap(); - let result = super::avx2_par4::authenticate_aead_par4(&aad, &ciphertext, &poly_key, lengths); + let lengths = AeadByteLengths::from_usize(aad.len(), ciphertext.len()); + // SAFETY: the test returned above unless AVX2 is available. + let result = unsafe { super::avx2_par4::authenticate_aead_par4(&aad, &ciphertext, &poly_key, lengths) }; assert_eq!(result, expected); } @@ -1202,7 +1437,7 @@ mod tests { 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, 0x06, 0x91, ]; - let lengths = AeadByteLengths::try_new(aad.len(), ciphertext.len()).unwrap(); + let lengths = AeadByteLengths::from_usize(aad.len(), ciphertext.len()); let result = super::aarch64_neon::authenticate_aead_par4(&aad, &ciphertext, &poly_key, lengths); assert_eq!(result, expected); } diff --git a/src/aead/poly1305/aarch64_neon.rs b/src/aead/poly1305/aarch64_neon.rs index bfa49861..1f6afac1 100644 --- a/src/aead/poly1305/aarch64_neon.rs +++ b/src/aead/poly1305/aarch64_neon.rs @@ -75,6 +75,12 @@ impl AeadPar4 { tag } + /// Absorbs one AEAD segment, padding its final partial block with zeros. + /// + /// # Safety + /// + /// The current CPU must support AArch64 NEON. `self` must retain the cache and power-table + /// invariants established by [`AeadPar4::new`]. #[target_feature(enable = "neon")] unsafe fn update_padded_segment_neon(&mut self, segment: &[u8]) { let mut offset = 0usize; @@ -92,8 +98,9 @@ impl AeadPar4 { for group in segment[offset..group_end].chunks_exact(64) { let (blocks, remainder) = group.as_chunks::<16>(); debug_assert!(remainder.is_empty()); + assert_eq!(blocks.len(), 4, "64-byte Poly1305 group must contain four blocks"); let [b0, b1, b2, b3] = blocks else { - unreachable!("64-byte Poly1305 group must split into four blocks"); + return; }; // SAFETY: direct four-block accumulation because: // 1. `group` is a 64-byte exact chunk split into four full Poly1305 blocks. @@ -121,6 +128,12 @@ impl AeadPar4 { } } + /// Caches one complete block and accumulates each full four-block group. + /// + /// # Safety + /// + /// The current CPU must support AArch64 NEON. `self` must retain the cache and power-table + /// invariants established by [`AeadPar4::new`], including `num_cached < 4` on entry. #[inline(always)] unsafe fn push(&mut self, block: [u8; 16]) { self.cached[self.num_cached] = block; @@ -133,7 +146,6 @@ impl AeadPar4 { } } -#[cfg_attr(not(any(feature = "xchacha20poly1305", feature = "diag", test)), allow(dead_code))] pub(super) fn authenticate_aead_par4( aad: &[u8], ciphertext: &[u8], @@ -146,12 +158,24 @@ pub(super) fn authenticate_aead_par4( authenticator.finalize(lengths) } +/// Accumulates four consecutive full blocks into `state`. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON, and `powers` must contain the first four powers of +/// the same clamped Poly1305 key represented by `state`. #[inline(always)] unsafe fn accumulate_4_blocks(blocks: &[[u8; 16]; 4], state: &mut State, powers: &Powers) { // SAFETY: caller is NEON-enabled and `blocks` contains four full 16-byte AEAD blocks. unsafe { accumulate_4_block_refs([&blocks[0], &blocks[1], &blocks[2], &blocks[3]], state, powers) }; } +/// Accumulates four referenced consecutive full blocks into `state`. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON, and `powers` must contain the first four powers of +/// the same clamped Poly1305 key represented by `state`. #[inline(always)] unsafe fn accumulate_4_block_refs(blocks: [&[u8; 16]; 4], state: &mut State, powers: &Powers) { let h = mul_unreduced(state.h, powers.r4); @@ -166,6 +190,12 @@ unsafe fn accumulate_4_block_refs(blocks: [&[u8; 16]; 4], state: &mut State, pow ]); } +/// Computes the unreduced four-block weighted sum for consecutive blocks. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON. `powers` must contain the first four powers of the +/// clamped Poly1305 key whose accumulator will receive the returned sum. #[inline(always)] unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; 5] { let b0 = block_limbs(blocks[0]); @@ -197,21 +227,21 @@ unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; lane4(r4[2], r3[2], r2[2], r1[2]), lane4(r4[3], r3[3], r2[3], r1[3]), lane4(r4[4], r3[4], r2[4], r1[4]), - lane4(r4[1] * 5, r3[1] * 5, r2[1] * 5, r1[1] * 5), - lane4(r4[2] * 5, r3[2] * 5, r2[2] * 5, r1[2] * 5), - lane4(r4[3] * 5, r3[3] * 5, r2[3] * 5, r1[3] * 5), - lane4(r4[4] * 5, r3[4] * 5, r2[4] * 5, r1[4] * 5), + lane4_fivefold(r4[1], r3[1], r2[1], r1[1]), + lane4_fivefold(r4[2], r3[2], r2[2], r1[2]), + lane4_fivefold(r4[3], r3[3], r2[3], r1[3]), + lane4_fivefold(r4[4], r3[4], r2[4], r1[4]), ) }; // SAFETY: all lanes are valid NEON vectors in this target-feature-enabled function. unsafe { [ - dot5_sum(x0, x1, x2, x3, x4, r0, s4, s3, s2, s1), - dot5_sum(x0, x1, x2, x3, x4, r1v, r0, s4, s3, s2), - dot5_sum(x0, x1, x2, x3, x4, r2v, r1v, r0, s4, s3), - dot5_sum(x0, x1, x2, x3, x4, r3v, r2v, r1v, r0, s4), - dot5_sum(x0, x1, x2, x3, x4, r4v, r3v, r2v, r1v, r0), + dot5_sum([x0, x1, x2, x3, x4], [r0, s4, s3, s2, s1]), + dot5_sum([x0, x1, x2, x3, x4], [r1v, r0, s4, s3, s2]), + dot5_sum([x0, x1, x2, x3, x4], [r2v, r1v, r0, s4, s3]), + dot5_sum([x0, x1, x2, x3, x4], [r3v, r2v, r1v, r0, s4]), + dot5_sum([x0, x1, x2, x3, x4], [r4v, r3v, r2v, r1v, r0]), ] } } @@ -227,6 +257,11 @@ fn block_limbs(block: &[u8; 16]) -> [u32; 5] { ] } +/// Packs four scalar values into consecutive NEON lanes. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON. #[inline(always)] unsafe fn lane4(a: u32, b: u32, c: u32, d: u32) -> uint32x4_t { // SAFETY: caller guarantees NEON is enabled; inputs are scalar lane values. @@ -238,20 +273,37 @@ unsafe fn lane4(a: u32, b: u32, c: u32, d: u32) -> uint32x4_t { } } +/// Multiplies four bounded limbs by five and packs them into consecutive NEON lanes. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON. Each input must be at most 858,993,459 so its +/// fivefold value fits in one `u32` lane. #[inline(always)] -#[allow(clippy::too_many_arguments)] -unsafe fn dot5_sum( - x0: uint32x4_t, - x1: uint32x4_t, - x2: uint32x4_t, - x3: uint32x4_t, - x4: uint32x4_t, - y0: uint32x4_t, - y1: uint32x4_t, - y2: uint32x4_t, - y3: uint32x4_t, - y4: uint32x4_t, -) -> u64 { +unsafe fn lane4_fivefold(a: u32, b: u32, c: u32, d: u32) -> uint32x4_t { + const MAX_UNSCALED: u32 = 858_993_459; + debug_assert!([a, b, c, d].into_iter().all(|limb| limb <= MAX_UNSCALED)); + + let a = u64::from(a).strict_mul(5); + let b = u64::from(b).strict_mul(5); + let c = u64::from(c).strict_mul(5); + let d = u64::from(d).strict_mul(5); + + // SAFETY: the caller guarantees NEON. The input bound proves every widened product fits in + // one 32-bit lane, so the two `u64` values encode four consecutive lanes without overlap. + unsafe { vcombine_u32(vcreate_u32((b << 32) | a), vcreate_u32((d << 32) | c)) } +} + +/// Sums five lane-wise products across four parallel Poly1305 streams. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON. +#[inline(always)] +unsafe fn dot5_sum(x: [uint32x4_t; 5], y: [uint32x4_t; 5]) -> u64 { + let [x0, x1, x2, x3, x4] = x; + let [y0, y1, y2, y3, y4] = y; + // SAFETY: caller guarantees NEON is enabled and all inputs are valid four-lane vectors. unsafe { let mut lo = vmull_u32(vget_low_u32(x0), vget_low_u32(y0)); @@ -266,6 +318,11 @@ unsafe fn dot5_sum( } } +/// Adds one lane-wise product to the low and high partial sums. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON. #[inline(always)] unsafe fn accumulate_mul(lo: &mut uint64x2_t, hi: &mut uint64x2_t, x: uint32x4_t, y: uint32x4_t) { // SAFETY: caller guarantees NEON is enabled and all inputs are valid four-lane vectors. @@ -282,40 +339,46 @@ fn mul_mod(a: [u32; 5], b: [u32; 5]) -> [u32; 5] { #[inline(always)] fn mul_unreduced(a: [u32; 5], b: [u32; 5]) -> [u64; 5] { - let b1_5 = b[1] * 5; - let b2_5 = b[2] * 5; - let b3_5 = b[3] * 5; - let b4_5 = b[4] * 5; + let b0 = u64::from(b[0]); + let b1 = u64::from(b[1]); + let b2 = u64::from(b[2]); + let b3 = u64::from(b[3]); + let b4 = u64::from(b[4]); + let b1_5 = b1.strict_mul(5); + let b2_5 = b2.strict_mul(5); + let b3_5 = b3.strict_mul(5); + let b4_5 = b4.strict_mul(5); [ - (u64::from(a[0]) * u64::from(b[0])) - + (u64::from(a[1]) * u64::from(b4_5)) - + (u64::from(a[2]) * u64::from(b3_5)) - + (u64::from(a[3]) * u64::from(b2_5)) - + (u64::from(a[4]) * u64::from(b1_5)), - (u64::from(a[0]) * u64::from(b[1])) - + (u64::from(a[1]) * u64::from(b[0])) - + (u64::from(a[2]) * u64::from(b4_5)) - + (u64::from(a[3]) * u64::from(b3_5)) - + (u64::from(a[4]) * u64::from(b2_5)), - (u64::from(a[0]) * u64::from(b[2])) - + (u64::from(a[1]) * u64::from(b[1])) - + (u64::from(a[2]) * u64::from(b[0])) - + (u64::from(a[3]) * u64::from(b4_5)) - + (u64::from(a[4]) * u64::from(b3_5)), - (u64::from(a[0]) * u64::from(b[3])) - + (u64::from(a[1]) * u64::from(b[2])) - + (u64::from(a[2]) * u64::from(b[1])) - + (u64::from(a[3]) * u64::from(b[0])) - + (u64::from(a[4]) * u64::from(b4_5)), - (u64::from(a[0]) * u64::from(b[4])) - + (u64::from(a[1]) * u64::from(b[3])) - + (u64::from(a[2]) * u64::from(b[2])) - + (u64::from(a[3]) * u64::from(b[1])) - + (u64::from(a[4]) * u64::from(b[0])), + scalar_dot5(a, [b0, b4_5, b3_5, b2_5, b1_5]), + scalar_dot5(a, [b1, b0, b4_5, b3_5, b2_5]), + scalar_dot5(a, [b2, b1, b0, b4_5, b3_5]), + scalar_dot5(a, [b3, b2, b1, b0, b4_5]), + scalar_dot5(a, [b4, b3, b2, b1, b0]), ] } +#[inline(always)] +fn scalar_dot5(a: [u32; 5], b: [u64; 5]) -> u64 { + let sum = u128::from(a[0]) + .strict_mul(u128::from(b[0])) + .strict_add(u128::from(a[1]).strict_mul(u128::from(b[1]))) + .strict_add(u128::from(a[2]).strict_mul(u128::from(b[2]))) + .strict_add(u128::from(a[3]).strict_mul(u128::from(b[3]))) + .strict_add(u128::from(a[4]).strict_mul(u128::from(b[4]))); + debug_assert!(sum <= u128::from(u64::MAX)); + + let [b0, b1, b2, b3, b4, b5, b6, b7, _, _, _, _, _, _, _, _] = sum.to_le_bytes(); + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]) +} + +#[inline(always)] +fn narrow_limb(value: u64) -> u32 { + debug_assert_eq!(value >> u32::BITS, 0); + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) +} + #[inline(always)] fn reduce_unreduced(mut d: [u64; 5]) -> [u32; 5] { let mut c = d[0] >> 26; @@ -327,20 +390,20 @@ fn reduce_unreduced(mut d: [u64; 5]) -> [u32; 5] { d[2] = d[2].wrapping_add(c); c = d[2] >> 26; - let h2 = (d[2] as u32) & LIMB_MASK; + let h2 = narrow_limb(d[2] & u64::from(LIMB_MASK)); d[3] = d[3].wrapping_add(c); c = d[3] >> 26; - let h3 = (d[3] as u32) & LIMB_MASK; + let h3 = narrow_limb(d[3] & u64::from(LIMB_MASK)); d[4] = d[4].wrapping_add(c); c = d[4] >> 26; - let h4 = (d[4] as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + let h4 = narrow_limb(d[4] & u64::from(LIMB_MASK)); + h0 = h0.wrapping_add(c.strict_mul(5)); c = h0 >> 26; h0 &= u64::from(LIMB_MASK); let h1 = h1_base.wrapping_add(c); - [h0 as u32, h1 as u32, h2, h3, h4] + [narrow_limb(h0), narrow_limb(h1), h2, h3, h4] } diff --git a/src/aead/poly1305/powerpc64_vsx.rs b/src/aead/poly1305/powerpc64_vsx.rs index af35330d..3e20829a 100644 --- a/src/aead/poly1305/powerpc64_vsx.rs +++ b/src/aead/poly1305/powerpc64_vsx.rs @@ -8,6 +8,11 @@ pub(super) fn compute_block(state: &mut State, block: &[u8; 16], partial: bool) unsafe { compute_block_impl(state, block, partial) } } +/// Process one Poly1305 block through the POWER8 vector multiplier. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec, VSX, and POWER8 vector instructions. #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn compute_block_impl(state: &mut State, block: &[u8; 16], partial: bool) { compute_block_scalar_reduction(state, block, partial, |lhs, rhs| { @@ -20,6 +25,10 @@ unsafe fn compute_block_impl(state: &mut State, block: &[u8; 16], partial: bool) /// /// Computes `lhs[0]*rhs[0] + lhs[1]*rhs[1] + lhs[2]*rhs[2] + lhs[3]*rhs[3]` /// using two 128-bit multiply-odd and one 128-bit add. +/// +/// # Safety +/// +/// The executing CPU must support POWER8 vector instructions. #[inline(always)] unsafe fn sum4_mul(lhs: [u32; 4], rhs: [u32; 4]) -> u64 { // SAFETY: POWER8+ VSX available via enclosing target_feature. @@ -34,11 +43,15 @@ unsafe fn sum4_mul(lhs: [u32; 4], rhs: [u32; 4]) -> u64 { let sum = vaddudm(prod_lo, prod_hi); let lanes = sum.to_array(); - (lanes[0] as u64).wrapping_add(lanes[1] as u64) + u64::from_ne_bytes(lanes[0].to_ne_bytes()).wrapping_add(u64::from_ne_bytes(lanes[1].to_ne_bytes())) } } /// Multiply low 32 bits of each u64 lane → u64: `vmulouw`. +/// +/// # Safety +/// +/// The executing CPU must support POWER8 vector instructions. #[inline(always)] unsafe fn vmulouw(a: i64x2, b: i64x2) -> i64x2 { let out: i64x2; @@ -56,6 +69,10 @@ unsafe fn vmulouw(a: i64x2, b: i64x2) -> i64x2 { } /// Add u64 lanes: `vaddudm`. +/// +/// # Safety +/// +/// The executing CPU must support POWER8 vector instructions. #[inline(always)] unsafe fn vaddudm(a: i64x2, b: i64x2) -> i64x2 { let out: i64x2; diff --git a/src/aead/poly1305/riscv64_vector.rs b/src/aead/poly1305/riscv64_vector.rs index dc27d658..2ee5e80c 100644 --- a/src/aead/poly1305/riscv64_vector.rs +++ b/src/aead/poly1305/riscv64_vector.rs @@ -1,4 +1,4 @@ -use core::simd::{i64x2, num::SimdUint, u32x4, u64x4}; +use core::simd::{num::SimdUint, u32x4, u64x2, u64x4}; use super::{FULL_BLOCK_HIBIT, LIMB_MASK, State, compute_block_scalar_reduction, load_u32_le}; use crate::{aead::AeadByteLengths, traits::ct}; @@ -9,6 +9,11 @@ pub(super) fn compute_block(state: &mut State, block: &[u8; 16], partial: bool) unsafe { compute_block_impl(state, block, partial) } } +/// Process one Poly1305 block through the RISC-V vector multiplier. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[target_feature(enable = "v")] unsafe fn compute_block_impl(state: &mut State, block: &[u8; 16], partial: bool) { compute_block_scalar_reduction(state, block, partial, |lhs, rhs| { @@ -18,19 +23,23 @@ unsafe fn compute_block_impl(state: &mut State, block: &[u8; 16], partial: bool) } /// Vectorized 4-element dot product using two 64-bit RVV lane multiplies. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[target_feature(enable = "v")] unsafe fn sum4_mul(lhs: [u32; 4], rhs: [u32; 4]) -> u64 { - let a_lo = i64x2::from_array([i64::from(lhs[0]), i64::from(lhs[1])]); - let b_lo = i64x2::from_array([i64::from(rhs[0]), i64::from(rhs[1])]); - let prod_lo = a_lo * b_lo; + let a_lo = u64x2::from_array([u64::from(lhs[0]), u64::from(lhs[1])]); + let b_lo = u64x2::from_array([u64::from(rhs[0]), u64::from(rhs[1])]); + let prod_lo = wrapping_mul2(a_lo, b_lo); - let a_hi = i64x2::from_array([i64::from(lhs[2]), i64::from(lhs[3])]); - let b_hi = i64x2::from_array([i64::from(rhs[2]), i64::from(rhs[3])]); - let prod_hi = a_hi * b_hi; + let a_hi = u64x2::from_array([u64::from(lhs[2]), u64::from(lhs[3])]); + let b_hi = u64x2::from_array([u64::from(rhs[2]), u64::from(rhs[3])]); + let prod_hi = wrapping_mul2(a_hi, b_hi); - let sum = prod_lo + prod_hi; + let sum = wrapping_add2(prod_lo, prod_hi); let lanes = sum.to_array(); - (lanes[0] as u64).wrapping_add(lanes[1] as u64) + lanes[0].wrapping_add(lanes[1]) } #[derive(Clone, Copy)] @@ -51,7 +60,6 @@ impl Powers { } } -#[cfg_attr(not(any(feature = "xchacha20poly1305", feature = "diag", test)), allow(dead_code))] pub(super) fn authenticate_aead_par4( aad: &[u8], ciphertext: &[u8], @@ -64,6 +72,12 @@ pub(super) fn authenticate_aead_par4( unsafe { authenticate_aead_par4_impl(aad, ciphertext, key, lengths) } } +/// Authenticate AEAD input four Poly1305 blocks at a time. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension, and `lengths` must +/// encode the byte lengths of `aad` and `ciphertext`. #[target_feature(enable = "v")] unsafe fn authenticate_aead_par4_impl( aad: &[u8], @@ -94,17 +108,13 @@ unsafe fn authenticate_aead_par4_impl( if num_cached == 0 { let group_len = segment.len().strict_sub(offset).strict_div(64).strict_mul(64); let group_end = offset.strict_add(group_len); - for group in segment[offset..group_end].chunks_exact(64) { - let (blocks, remainder) = group.as_chunks::<16>(); - debug_assert!(remainder.is_empty()); - let [b0, b1, b2, b3] = blocks else { - unreachable!("64-byte Poly1305 group must split into four blocks"); - }; + for group in segment[offset..group_end].as_chunks::<64>().0 { + let blocks = group.as_chunks::<16>().0; // SAFETY: direct four-block accumulation because: // 1. `group` is a 64-byte exact chunk split into four full Poly1305 blocks. // 2. This entry point is compiled with the RISC-V V target feature. // 3. `num_cached == 0`, so direct accumulation preserves AEAD block order. - unsafe { accumulate_4_block_refs([b0, b1, b2, b3], &mut state, &powers) }; + unsafe { accumulate_4_block_refs([&blocks[0], &blocks[1], &blocks[2], &blocks[3]], &mut state, &powers) }; } offset = group_end; } @@ -154,6 +164,12 @@ unsafe fn authenticate_aead_par4_impl( tag } +/// Queue one full Poly1305 block and fold the queue when it reaches four blocks. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension, and `*num_cached` +/// must be less than four. #[inline(always)] unsafe fn push_cached( cached: &mut [[u8; 16]; 4], @@ -173,6 +189,11 @@ unsafe fn push_cached( } } +/// Accumulate four consecutive Poly1305 blocks. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[inline(always)] unsafe fn accumulate_4_blocks(blocks: &[[u8; 16]; 4], state: &mut State, powers: &Powers) { // SAFETY: RVV four-block accumulation because: @@ -181,6 +202,11 @@ unsafe fn accumulate_4_blocks(blocks: &[[u8; 16]; 4], state: &mut State, powers: unsafe { accumulate_4_block_refs([&blocks[0], &blocks[1], &blocks[2], &blocks[3]], state, powers) }; } +/// Accumulate four consecutive Poly1305 blocks supplied by reference. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[inline(always)] unsafe fn accumulate_4_block_refs(blocks: [&[u8; 16]; 4], state: &mut State, powers: &Powers) { let h = mul_unreduced(state.h, powers.r4); @@ -197,6 +223,11 @@ unsafe fn accumulate_4_block_refs(blocks: [&[u8; 16]; 4], state: &mut State, pow ]); } +/// Multiply four spaced messages by descending powers of the Poly1305 key. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[inline(always)] unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; 5] { let b0 = block_limbs(blocks[0]); @@ -241,7 +272,7 @@ unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; r2[1].wrapping_mul(5), r1[1].wrapping_mul(5), ); - dot5_sum(x0, x1, x2, x3, x4, r0, s4, s3, s2, s1) + dot5_sum([x0, x1, x2, x3, x4], [r0, s4, s3, s2, s1]) }; let d1 = { let r1v = lane4(r4[1], r3[1], r2[1], r1[1]); @@ -264,7 +295,7 @@ unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; r2[2].wrapping_mul(5), r1[2].wrapping_mul(5), ); - dot5_sum(x0, x1, x2, x3, x4, r1v, r0, s4, s3, s2) + dot5_sum([x0, x1, x2, x3, x4], [r1v, r0, s4, s3, s2]) }; let d2 = { let r2v = lane4(r4[2], r3[2], r2[2], r1[2]); @@ -282,7 +313,7 @@ unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; r2[3].wrapping_mul(5), r1[3].wrapping_mul(5), ); - dot5_sum(x0, x1, x2, x3, x4, r2v, r1v, r0, s4, s3) + dot5_sum([x0, x1, x2, x3, x4], [r2v, r1v, r0, s4, s3]) }; let d3 = { let r3v = lane4(r4[3], r3[3], r2[3], r1[3]); @@ -295,7 +326,7 @@ unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; r2[4].wrapping_mul(5), r1[4].wrapping_mul(5), ); - dot5_sum(x0, x1, x2, x3, x4, r3v, r2v, r1v, r0, s4) + dot5_sum([x0, x1, x2, x3, x4], [r3v, r2v, r1v, r0, s4]) }; let d4 = { let r4v = lane4(r4[4], r3[4], r2[4], r1[4]); @@ -303,7 +334,7 @@ unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; let r2v = lane4(r4[2], r3[2], r2[2], r1[2]); let r1v = lane4(r4[1], r3[1], r2[1], r1[1]); let r0 = lane4(r4[0], r3[0], r2[0], r1[0]); - dot5_sum(x0, x1, x2, x3, x4, r4v, r3v, r2v, r1v, r0) + dot5_sum([x0, x1, x2, x3, x4], [r4v, r3v, r2v, r1v, r0]) }; [d0, d1, d2, d3, d4] @@ -331,24 +362,32 @@ fn widen(value: u32x4) -> u64x4 { } #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn dot5_sum( - x0: u32x4, - x1: u32x4, - x2: u32x4, - x3: u32x4, - x4: u32x4, - y0: u32x4, - y1: u32x4, - y2: u32x4, - y3: u32x4, - y4: u32x4, -) -> u64 { - let sum = widen(x0) * widen(y0) - + widen(x1) * widen(y1) - + widen(x2) * widen(y2) - + widen(x3) * widen(y3) - + widen(x4) * widen(y4); +fn wrapping_mul2(lhs: u64x2, rhs: u64x2) -> u64x2 { + core::ops::Mul::mul(lhs, rhs) +} + +#[inline(always)] +fn wrapping_add2(lhs: u64x2, rhs: u64x2) -> u64x2 { + core::ops::Add::add(lhs, rhs) +} + +#[inline(always)] +fn wrapping_mul4(lhs: u64x4, rhs: u64x4) -> u64x4 { + core::ops::Mul::mul(lhs, rhs) +} + +#[inline(always)] +fn wrapping_add4(lhs: u64x4, rhs: u64x4) -> u64x4 { + core::ops::Add::add(lhs, rhs) +} + +#[inline(always)] +fn dot5_sum(x: [u32x4; 5], y: [u32x4; 5]) -> u64 { + let mut sum = wrapping_mul4(widen(x[0]), widen(y[0])); + sum = wrapping_add4(sum, wrapping_mul4(widen(x[1]), widen(y[1]))); + sum = wrapping_add4(sum, wrapping_mul4(widen(x[2]), widen(y[2]))); + sum = wrapping_add4(sum, wrapping_mul4(widen(x[3]), widen(y[3]))); + sum = wrapping_add4(sum, wrapping_mul4(widen(x[4]), widen(y[4]))); let lanes = sum.to_array(); lanes[0] .wrapping_add(lanes[1]) @@ -399,20 +438,26 @@ fn reduce_unreduced(mut d: [u64; 5]) -> [u32; 5] { d[2] = d[2].wrapping_add(c); c = d[2] >> 26; - let h2 = (d[2] as u32) & LIMB_MASK; + let h2 = low_u32(d[2]) & LIMB_MASK; d[3] = d[3].wrapping_add(c); c = d[3] >> 26; - let h3 = (d[3] as u32) & LIMB_MASK; + let h3 = low_u32(d[3]) & LIMB_MASK; d[4] = d[4].wrapping_add(c); c = d[4] >> 26; - let h4 = (d[4] as u32) & LIMB_MASK; + let h4 = low_u32(d[4]) & LIMB_MASK; h0 = h0.wrapping_add(c.wrapping_mul(5)); c = h0 >> 26; h0 &= u64::from(LIMB_MASK); let h1 = h1_base.wrapping_add(c); - [h0 as u32, h1 as u32, h2, h3, h4] + [low_u32(h0), low_u32(h1), h2, h3, h4] +} + +#[inline(always)] +fn low_u32(value: u64) -> u32 { + let bytes = value.to_le_bytes(); + u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) } diff --git a/src/aead/poly1305/s390x_vector.rs b/src/aead/poly1305/s390x_vector.rs index df8ae069..f8e3cfaa 100644 --- a/src/aead/poly1305/s390x_vector.rs +++ b/src/aead/poly1305/s390x_vector.rs @@ -8,6 +8,11 @@ pub(super) fn compute_block(state: &mut State, block: &[u8; 16], partial: bool) unsafe { compute_block_impl(state, block, partial) } } +/// Process one Poly1305 block through the s390x vector multiplier. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[target_feature(enable = "vector")] unsafe fn compute_block_impl(state: &mut State, block: &[u8; 16], partial: bool) { compute_block_scalar_reduction(state, block, partial, |lhs, rhs| { @@ -20,6 +25,10 @@ unsafe fn compute_block_impl(state: &mut State, block: &[u8; 16], partial: bool) /// /// Computes `lhs[0]*rhs[0] + lhs[1]*rhs[1] + lhs[2]*rhs[2] + lhs[3]*rhs[3]` /// using two 128-bit multiply-odd and one 128-bit add. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn sum4_mul(lhs: [u32; 4], rhs: [u32; 4]) -> u64 { @@ -35,11 +44,15 @@ unsafe fn sum4_mul(lhs: [u32; 4], rhs: [u32; 4]) -> u64 { let sum = vag(prod_lo, prod_hi); let lanes = sum.to_array(); - (lanes[0] as u64).wrapping_add(lanes[1] as u64) + u64::from_ne_bytes(lanes[0].to_ne_bytes()).wrapping_add(u64::from_ne_bytes(lanes[1].to_ne_bytes())) } } /// Multiply odd-indexed u32 lanes → u64: `vmlof`. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn vmlof(a: i64x2, b: i64x2) -> i64x2 { @@ -58,6 +71,10 @@ unsafe fn vmlof(a: i64x2, b: i64x2) -> i64x2 { } /// Add u64 lanes: `vag`. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn vag(a: i64x2, b: i64x2) -> i64x2 { diff --git a/src/aead/poly1305/x86_64_avx2_par4.rs b/src/aead/poly1305/x86_64_avx2_par4.rs index 406fb48a..048d4722 100644 --- a/src/aead/poly1305/x86_64_avx2_par4.rs +++ b/src/aead/poly1305/x86_64_avx2_par4.rs @@ -60,25 +60,40 @@ struct SpacedMultiplier4x130 { impl Aligned130 { /// Pack five scalar 26-bit limbs into a `__m256i`. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn from_limbs(limbs: [u32; 5]) -> Self { - Aligned130(_mm256_setr_epi32( - limbs[0] as i32, - limbs[1] as i32, - limbs[2] as i32, - limbs[3] as i32, - limbs[4] as i32, - 0, - 0, - 0, - )) + // SAFETY: the caller guarantees AVX2 support. + let lanes = unsafe { + _mm256_setr_epi32( + limbs[0].cast_signed(), + limbs[1].cast_signed(), + limbs[2].cast_signed(), + limbs[3].cast_signed(), + limbs[4].cast_signed(), + 0, + 0, + 0, + ) + }; + Aligned130(lanes) } /// Extract five scalar 26-bit limbs. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn into_limbs(self) -> [u32; 5] { let mut buf = [0u32; 8]; - _mm256_storeu_si256(buf.as_mut_ptr() as *mut __m256i, self.0); + let dst = core::ptr::NonNull::from(&mut buf).cast::<__m256i>().as_ptr(); + // SAFETY: the caller guarantees AVX2 support. `dst` retains `buf`'s provenance and points to its entire writable + // 32-byte range; the unaligned store has no stronger alignment requirement. + unsafe { _mm256_storeu_si256(dst, self.0) }; [buf[0], buf[1], buf[2], buf[3], buf[4]] } @@ -86,187 +101,282 @@ impl Aligned130 { /// /// AEAD-only: unconditionally sets the 2¹²⁸ high bit. Not suitable for raw /// Poly1305 where partial blocks omit the hibit. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn from_block(block: &[u8; 16]) -> Self { - Self::split_to_26bit(_mm256_or_si256( - _mm256_and_si256( - _mm256_castsi128_si256(_mm_loadu_si128(block.as_ptr() as *const _)), - _mm256_set_epi64x(0, 0, -1, -1), - ), - _mm256_set_epi64x(0, 1, 0, 0), - )) + let src = core::ptr::NonNull::from(block).cast::<__m128i>().as_ptr(); + // SAFETY: the caller guarantees AVX2 support. `src` retains `block`'s provenance and exposes its complete + // initialized 16-byte range; the load is unaligned. The nested split performs only AVX2 register operations. + unsafe { + Self::split_to_26bit(_mm256_or_si256( + _mm256_and_si256( + _mm256_castsi128_si256(_mm_loadu_si128(src)), + _mm256_set_epi64x(0, 0, -1, -1), + ), + _mm256_set_epi64x(0, 1, 0, 0), + )) + } } /// Split a 130-bit integer (low 5 words) into 26-bit limbs. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn split_to_26bit(x: __m256i) -> Self { - let xl = _mm256_sllv_epi32(x, _mm256_set_epi32(32, 32, 32, 24, 18, 12, 6, 0)); - let xh = _mm256_permutevar8x32_epi32( - _mm256_srlv_epi32(x, _mm256_set_epi32(32, 32, 32, 2, 8, 14, 20, 26)), - _mm256_set_epi32(6, 5, 4, 3, 2, 1, 0, 7), - ); - Aligned130(_mm256_and_si256( - _mm256_or_si256(xl, xh), - _mm256_set_epi32(0, 0, 0, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff), - )) + // SAFETY: the caller guarantees AVX2 support; all operations are register-only. + unsafe { + let xl = _mm256_sllv_epi32(x, _mm256_set_epi32(32, 32, 32, 24, 18, 12, 6, 0)); + let xh = _mm256_permutevar8x32_epi32( + _mm256_srlv_epi32(x, _mm256_set_epi32(32, 32, 32, 2, 8, 14, 20, 26)), + _mm256_set_epi32(6, 5, 4, 3, 2, 1, 0, 7), + ); + Aligned130(_mm256_and_si256( + _mm256_or_si256(xl, xh), + _mm256_set_epi32(0, 0, 0, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff), + )) + } } + /// Add two aligned values independently in each 32-bit lane. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn add(self, other: Aligned130) -> Aligned130 { - Aligned130(_mm256_add_epi32(self.0, other.0)) + // SAFETY: the caller guarantees AVX2 support. + Aligned130(unsafe { _mm256_add_epi32(self.0, other.0) }) } } // ── PrecomputedMultiplier ──────────────────────────────────────────────── impl PrecomputedMultiplier { + /// Precompute the lane-packed multiplier from the kernel's bounded radix-2²⁶ representation. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn from_aligned(r: Aligned130) -> Self { - // 5*R limbs: r + (r << 2) = r * 5 - let a_5 = _mm256_permutevar8x32_epi32( - _mm256_add_epi32(r.0, _mm256_slli_epi32(r.0, 2)), - _mm256_set_epi32(4, 3, 2, 1, 1, 1, 1, 1), - ); - let a = _mm256_blend_epi32(r.0, a_5, 0b11100000); - let a_5 = _mm256_permute2x128_si256(a_5, a_5, 0); - PrecomputedMultiplier { a, a_5 } + // SAFETY: the caller guarantees AVX2 support; all operations are register-only. + unsafe { + // 5*R limbs: r + (r << 2) = r * 5 + let a_5 = _mm256_permutevar8x32_epi32( + _mm256_add_epi32(r.0, _mm256_slli_epi32(r.0, 2)), + _mm256_set_epi32(4, 3, 2, 1, 1, 1, 1, 1), + ); + let a = _mm256_blend_epi32(r.0, a_5, 0b11100000); + let a_5 = _mm256_permute2x128_si256(a_5, a_5, 0); + PrecomputedMultiplier { a, a_5 } + } } } // ── Single multiply: Aligned130 × PrecomputedMultiplier → Unreduced130 ── +/// Multiply one bounded radix-2²⁶ value by a precomputed multiplier. +/// +/// The low five lanes of `x` must be below 2²⁸, and `r` must be derived from reduced multiplier limbs below 2²⁷. +/// These bounds keep the precomputed fivefold lanes below 2³⁰ and every five-product coefficient below 2⁶¹. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn mul_single(x: Aligned130, r: PrecomputedMultiplier) -> Unreduced130 { - let x = x.0; - let y = r.a; - let z = r.a_5; - - // v0 = [t3, t2, t1, t0] — accumulate 5 products per limb. - let mut v0 = _mm256_mul_epu32( - _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(4, 3, 2, 1)), - _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(7, 7, 7, 7)), - ); - v0 = _mm256_add_epi64( - v0, - _mm256_mul_epu32( + // SAFETY: the caller guarantees AVX2 support; the documented bounds keep every 64-bit lane sum below 2^61. + unsafe { + let x = x.0; + let y = r.a; + let z = r.a_5; + + // v0 = [t3, t2, t1, t0] — accumulate 5 products per limb. + let mut v0 = _mm256_mul_epu32( + _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(4, 3, 2, 1)), + _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(7, 7, 7, 7)), + ); + v0 = _mm256_add_epi64( + v0, + _mm256_mul_epu32( + _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(3, 2, 1, 0)), + _mm256_broadcastd_epi32(_mm256_castsi256_si128(y)), + ), + ); + v0 = _mm256_add_epi64( + v0, + _mm256_mul_epu32( + _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(1, 1, 3, 3)), + _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(2, 1, 6, 5)), + ), + ); + v0 = _mm256_add_epi64( + v0, + _mm256_mul_epu32( + _mm256_permute4x64_epi64(x, imm8(1, 0, 0, 2)), + _mm256_blend_epi32(_mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(1, 2, 1, 1)), z, 0x03), + ), + ); + v0 = _mm256_add_epi64( + v0, + _mm256_mul_epu32( + _mm256_permute4x64_epi64(x, imm8(0, 2, 2, 1)), + _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(3, 6, 5, 6)), + ), + ); + + // v1 = [_, _, _, t4] + let mut v1 = _mm256_mul_epu32( _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(3, 2, 1, 0)), - _mm256_broadcastd_epi32(_mm256_castsi256_si128(y)), - ), - ); - v0 = _mm256_add_epi64( - v0, - _mm256_mul_epu32( - _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(1, 1, 3, 3)), - _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(2, 1, 6, 5)), - ), - ); - v0 = _mm256_add_epi64( - v0, - _mm256_mul_epu32( - _mm256_permute4x64_epi64(x, imm8(1, 0, 0, 2)), - _mm256_blend_epi32(_mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(1, 2, 1, 1)), z, 0x03), - ), - ); - v0 = _mm256_add_epi64( - v0, - _mm256_mul_epu32( - _mm256_permute4x64_epi64(x, imm8(0, 2, 2, 1)), - _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(3, 6, 5, 6)), - ), - ); - - // v1 = [_, _, _, t4] - let mut v1 = _mm256_mul_epu32( - _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(3, 2, 1, 0)), - _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(1, 2, 3, 4)), - ); - v1 = _mm256_add_epi64(v1, _mm256_permute4x64_epi64(v1, imm8(1, 0, 3, 2))); - v1 = _mm256_add_epi64(v1, _mm256_permute4x64_epi64(v1, imm8(0, 0, 0, 1))); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(_mm256_permute4x64_epi64(x, imm8(0, 0, 0, 2)), y)); - - Unreduced130 { v0, v1 } + _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(1, 2, 3, 4)), + ); + v1 = _mm256_add_epi64(v1, _mm256_permute4x64_epi64(v1, imm8(1, 0, 3, 2))); + v1 = _mm256_add_epi64(v1, _mm256_permute4x64_epi64(v1, imm8(0, 0, 0, 1))); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(_mm256_permute4x64_epi64(x, imm8(0, 0, 0, 2)), y)); + + Unreduced130 { v0, v1 } + } } // ── Unreduced130 carry chain and reduction ─────────────────────────────── /// Carry: propagate bits >26 from v0 into v1. +/// +/// The inputs must be bounded unreduced coefficients produced by this kernel's multiplication or preceding fold. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn adc_single(v1: __m256i, v0: __m256i) -> (__m256i, __m256i) { - let v0 = _mm256_add_epi64( - _mm256_and_si256(v0, _mm256_set_epi64x(-1, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff)), - _mm256_permute4x64_epi64( - _mm256_srlv_epi64(v0, _mm256_set_epi64x(64, 26, 26, 26)), - imm8(2, 1, 0, 3), - ), - ); - let v1 = _mm256_add_epi64( - v1, - _mm256_permute4x64_epi64(_mm256_srli_epi64(v0, 26), imm8(2, 1, 0, 3)), - ); - let chain = _mm256_and_si256(v0, _mm256_set_epi64x(0x3ff_ffff, -1, -1, -1)); - (v1, chain) + // SAFETY: the caller guarantees AVX2 support; the kernel bounds keep every propagated carry and lane sum in u64. + unsafe { + let v0 = _mm256_add_epi64( + _mm256_and_si256(v0, _mm256_set_epi64x(-1, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff)), + _mm256_permute4x64_epi64( + _mm256_srlv_epi64(v0, _mm256_set_epi64x(64, 26, 26, 26)), + imm8(2, 1, 0, 3), + ), + ); + let v1 = _mm256_add_epi64( + v1, + _mm256_permute4x64_epi64(_mm256_srli_epi64(v0, 26), imm8(2, 1, 0, 3)), + ); + let chain = _mm256_and_si256(v0, _mm256_set_epi64x(0x3ff_ffff, -1, -1, -1)); + (v1, chain) + } } /// Reduce modulo 2¹³⁰ − 5: fold top limb back into bottom. +/// +/// The inputs must be the bounded carry-propagated representation returned by [`adc_single`]. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn red_single(v1: __m256i, v0: __m256i) -> (__m256i, __m256i) { - let t = _mm256_srlv_epi64(v1, _mm256_set_epi64x(64, 64, 64, 26)); - let red_0 = _mm256_add_epi64(_mm256_add_epi64(v0, t), _mm256_slli_epi64(t, 2)); - let red_1 = _mm256_and_si256(v1, _mm256_set_epi64x(0, 0, 0, 0x3ff_ffff)); - (red_1, red_0) + // SAFETY: the caller guarantees AVX2 support; the bounded top carry and its fivefold fold fit in every u64 lane. + unsafe { + let t = _mm256_srlv_epi64(v1, _mm256_set_epi64x(64, 64, 64, 26)); + let red_0 = _mm256_add_epi64(_mm256_add_epi64(v0, t), _mm256_slli_epi64(t, 2)); + let red_1 = _mm256_and_si256(v1, _mm256_set_epi64x(0, 0, 0, 0x3ff_ffff)); + (red_1, red_0) + } } impl Unreduced130 { + /// Carry-reduce this bounded convolution result into the kernel's radix-2²⁶ representation. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn reduce(self) -> Aligned130 { - let (v1, v0) = adc_single(self.v1, self.v0); - let (v1, v0) = red_single(v1, v0); - let (v1, v0) = adc_single(v1, v0); - // Switch from 64-bit to 32-bit limbs. - Aligned130(_mm256_blend_epi32( - _mm256_permutevar8x32_epi32(v0, _mm256_set_epi32(0, 6, 4, 0, 6, 4, 2, 0)), - _mm256_permutevar8x32_epi32(v1, _mm256_set_epi32(0, 6, 4, 0, 6, 4, 2, 0)), - 0x90, - )) + // SAFETY: the caller guarantees AVX2 support. `self` is a bounded kernel convolution, satisfying both nested + // helpers' carry/fold contracts; the final operations only repack the resulting register lanes. + unsafe { + let (v1, v0) = adc_single(self.v1, self.v0); + let (v1, v0) = red_single(v1, v0); + let (v1, v0) = adc_single(v1, v0); + // Switch from 64-bit to 32-bit limbs. + Aligned130(_mm256_blend_epi32( + _mm256_permutevar8x32_epi32(v0, _mm256_set_epi32(0, 6, 4, 0, 6, 4, 2, 0)), + _mm256_permutevar8x32_epi32(v1, _mm256_set_epi32(0, 6, 4, 0, 6, 4, 2, 0)), + 0x90, + )) + } } } // ── Aligned4x130 ──────────────────────────────────────────────────────── impl Aligned4x130 { + /// Load four consecutive full blocks for parallel processing. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn from_blocks(src: &[[u8; 16]; 4]) -> Self { - // SAFETY: `[[u8; 16]; 4]` is 64 contiguous bytes; two 32-byte loads are valid. - let ptr = src.as_ptr() as *const __m256i; - let blocks_01 = _mm256_loadu_si256(ptr); - let blocks_23 = _mm256_loadu_si256(ptr.add(1)); - Self::from_loaded_blocks(blocks_01, blocks_23) + let ptr = core::ptr::NonNull::from(src).cast::<__m256i>().as_ptr(); + // SAFETY: the caller guarantees AVX2 support. `ptr` retains `src`'s provenance; `src` contains 64 initialized + // readable bytes, so the unaligned 32-byte loads at offsets 0 and 32 are valid. The nested helper is register-only. + unsafe { + let blocks_01 = _mm256_loadu_si256(ptr); + let blocks_23 = _mm256_loadu_si256(ptr.add(1)); + Self::from_loaded_blocks(blocks_01, blocks_23) + } } /// Interleave 4 blocks into 20 packed 26-bit limbs across 3 vectors. + /// + /// `blocks_01` and `blocks_23` must contain four consecutive full blocks in that order. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn from_loaded_blocks(blocks_01: __m256i, blocks_23: __m256i) -> Self { - let mask_26 = _mm256_set1_epi32(0x3ff_ffff); - let set_hibit = _mm256_set1_epi32(1 << 24); + // SAFETY: the caller guarantees AVX2 support; all operations are register-only. + unsafe { + let mask_26 = _mm256_set1_epi32(0x3ff_ffff); + let set_hibit = _mm256_set1_epi32(1 << 24); - let a0 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi64(blocks_01, blocks_23), imm8(3, 1, 2, 0)); - let a1 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi64(blocks_01, blocks_23), imm8(3, 1, 2, 0)); + let a0 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi64(blocks_01, blocks_23), imm8(3, 1, 2, 0)); + let a1 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi64(blocks_01, blocks_23), imm8(3, 1, 2, 0)); - let v2 = _mm256_or_si256(_mm256_srli_epi64(a0, 40), set_hibit); - let a2 = _mm256_or_si256(_mm256_srli_epi64(a1, 46), _mm256_slli_epi64(a0, 18)); + let v2 = _mm256_or_si256(_mm256_srli_epi64(a0, 40), set_hibit); + let a2 = _mm256_or_si256(_mm256_srli_epi64(a1, 46), _mm256_slli_epi64(a0, 18)); - let v1 = _mm256_and_si256(_mm256_blend_epi32(_mm256_srli_epi64(a1, 26), a2, 0xAA), mask_26); - let v0 = _mm256_and_si256(_mm256_blend_epi32(a1, _mm256_slli_epi64(a2, 26), 0xAA), mask_26); + let v1 = _mm256_and_si256(_mm256_blend_epi32(_mm256_srli_epi64(a1, 26), a2, 0xAA), mask_26); + let v0 = _mm256_and_si256(_mm256_blend_epi32(a1, _mm256_slli_epi64(a2, 26), 0xAA), mask_26); - Aligned4x130 { v0, v1, v2 } + Aligned4x130 { v0, v1, v2 } + } } + /// Add four aligned values independently in each 32-bit lane. + /// + /// Both inputs must satisfy the kernel's loose-limb bounds, which keep every lane sum in `u32`. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn add(self, other: Aligned4x130) -> Aligned4x130 { - Aligned4x130 { - v0: _mm256_add_epi32(self.v0, other.v0), - v1: _mm256_add_epi32(self.v1, other.v1), - v2: _mm256_add_epi32(self.v2, other.v2), + // SAFETY: the caller guarantees AVX2 support; the documented bounds keep all three lane sums in u32. + unsafe { + Aligned4x130 { + v0: _mm256_add_epi32(self.v0, other.v0), + v1: _mm256_add_epi32(self.v1, other.v1), + v2: _mm256_add_epi32(self.v2, other.v2), + } } } } @@ -274,203 +384,243 @@ impl Aligned4x130 { // ── 4-way parallel multiply ────────────────────────────────────────────── /// Multiply 4 values by the same R: `(x0·R, x1·R, x2·R, x3·R)`. +/// +/// `x` and `r` must satisfy the kernel's loose-radix bounds. Those bounds keep every five-product coefficient below +/// 2⁶¹. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn mul_4x130(x: &Aligned4x130, r: PrecomputedMultiplier) -> Unreduced4x130 { - let mut x = *x; - let y = r.a; - let z = r.a_5; - let ord = _mm256_set_epi32(6, 7, 4, 5, 2, 3, 0, 1); + // SAFETY: the caller guarantees AVX2 support; the documented bounds keep every 64-bit lane sum below 2^61. + unsafe { + let mut x = *x; + let y = r.a; + let z = r.a_5; + let ord = _mm256_set_epi32(6, 7, 4, 5, 2, 3, 0, 1); - let mut t0 = _mm256_permute4x64_epi64(y, imm8(0, 0, 0, 0)); - let mut t1 = _mm256_permute4x64_epi64(y, imm8(1, 1, 1, 1)); + let mut t0 = _mm256_permute4x64_epi64(y, imm8(0, 0, 0, 0)); + let mut t1 = _mm256_permute4x64_epi64(y, imm8(1, 1, 1, 1)); - let mut v0 = _mm256_mul_epu32(x.v0, t0); - let mut v1 = _mm256_mul_epu32(x.v1, t0); - let mut v4 = _mm256_mul_epu32(x.v2, t0); - let mut v2 = _mm256_mul_epu32(x.v0, t1); - let mut v3 = _mm256_mul_epu32(x.v1, t1); + let mut v0 = _mm256_mul_epu32(x.v0, t0); + let mut v1 = _mm256_mul_epu32(x.v1, t0); + let mut v4 = _mm256_mul_epu32(x.v2, t0); + let mut v2 = _mm256_mul_epu32(x.v0, t1); + let mut v3 = _mm256_mul_epu32(x.v1, t1); - t0 = _mm256_permutevar8x32_epi32(t0, ord); - t1 = _mm256_permutevar8x32_epi32(t1, ord); + t0 = _mm256_permutevar8x32_epi32(t0, ord); + t1 = _mm256_permutevar8x32_epi32(t1, ord); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, t0)); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, t0)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, t1)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, t1)); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, t0)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, t0)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, t1)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, t1)); - let mut t2 = _mm256_permute4x64_epi64(y, imm8(2, 2, 2, 2)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v0, t2)); + let mut t2 = _mm256_permute4x64_epi64(y, imm8(2, 2, 2, 2)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v0, t2)); - x.v0 = _mm256_permutevar8x32_epi32(x.v0, ord); - x.v1 = _mm256_permutevar8x32_epi32(x.v1, ord); - t2 = _mm256_permutevar8x32_epi32(t2, ord); + x.v0 = _mm256_permutevar8x32_epi32(x.v0, ord); + x.v1 = _mm256_permutevar8x32_epi32(x.v1, ord); + t2 = _mm256_permutevar8x32_epi32(t2, ord); - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, t2)); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v2, t2)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, t0)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, t0)); + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, t2)); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v2, t2)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, t0)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, t0)); - t0 = _mm256_permutevar8x32_epi32(t0, ord); - t1 = _mm256_permutevar8x32_epi32(t1, ord); + t0 = _mm256_permutevar8x32_epi32(t0, ord); + t1 = _mm256_permutevar8x32_epi32(t1, ord); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v0, t0)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v1, t0)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v0, t1)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v0, t0)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v1, t0)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v0, t1)); - t0 = _mm256_permute4x64_epi64(y, imm8(3, 3, 3, 3)); + t0 = _mm256_permute4x64_epi64(y, imm8(3, 3, 3, 3)); - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v0, t0)); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v1, t0)); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v2, t0)); + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v0, t0)); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v1, t0)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v2, t0)); - t0 = _mm256_permutevar8x32_epi32(t0, ord); + t0 = _mm256_permutevar8x32_epi32(t0, ord); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, t0)); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, t0)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v2, t0)); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, t0)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, t0)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v2, t0)); - x.v1 = _mm256_permutevar8x32_epi32(x.v1, ord); + x.v1 = _mm256_permutevar8x32_epi32(x.v1, ord); - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, t0)); - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v2, z)); + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, t0)); + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v2, z)); - Unreduced4x130 { v0, v1, v2, v3, v4 } + Unreduced4x130 { v0, v1, v2, v3, v4 } + } } // ── Spaced multiply ───────────────────────────────────────────────────── /// Multiply lane i by R^(4−i): `(x0·R⁴, x1·R³, x2·R², x3·R¹)`. +/// +/// `x` and `m` must satisfy the kernel's loose-radix bounds and represent matching streams and powers of one key. +/// Those bounds keep every five-product coefficient below 2⁶¹. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn mul_spaced(x: Aligned4x130, m: SpacedMultiplier4x130) -> Unreduced4x130 { - let mut x = x; - let r1 = m.r1.a; - - let v0u = _mm256_unpacklo_epi32(m.v0, m.v1); - let v1u = _mm256_unpackhi_epi32(m.v0, m.v1); - - let ord_a = _mm256_set_epi32(1, 0, 6, 7, 2, 0, 3, 1); - let m_r_0 = _mm256_blend_epi32( - _mm256_permutevar8x32_epi32(r1, ord_a), - _mm256_permutevar8x32_epi32(v0u, ord_a), - 0b00111111, - ); - let ord_b = _mm256_set_epi32(3, 2, 4, 5, 2, 0, 3, 1); - let m_r_2 = _mm256_blend_epi32( - _mm256_permutevar8x32_epi32(r1, ord_b), - _mm256_permutevar8x32_epi32(v1u, ord_b), - 0b00111111, - ); - let ord_c = _mm256_set_epi32(1, 4, 6, 6, 2, 4, 3, 5); - let m_r_4 = _mm256_blend_epi32( - _mm256_blend_epi32( - _mm256_permutevar8x32_epi32(r1, ord_c), - _mm256_permutevar8x32_epi32(v1u, ord_c), - 0b00010000, - ), - _mm256_permutevar8x32_epi32(v0u, ord_c), - 0b00101111, - ); - - let mut v0 = _mm256_mul_epu32(x.v0, m_r_0); - let mut v1 = _mm256_mul_epu32(x.v1, m_r_0); - let mut v2 = _mm256_mul_epu32(x.v0, m_r_2); - let mut v3 = _mm256_mul_epu32(x.v1, m_r_2); - let mut v4 = _mm256_mul_epu32(x.v0, m_r_4); - - let swap = _mm256_set_epi32(6, 7, 4, 5, 2, 3, 0, 1); - let m_r_1 = _mm256_permutevar8x32_epi32(m_r_0, swap); - let m_r_3 = _mm256_permutevar8x32_epi32(m_r_2, swap); - - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, m_r_1)); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, m_r_1)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, m_r_3)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, m_r_3)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v2, m_r_0)); - - x.v0 = _mm256_permutevar8x32_epi32(x.v0, swap); - - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v0, m_r_0)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, m_r_1)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v0, m_r_2)); - - let m_5r_3 = _mm256_add_epi32(m_r_3, _mm256_slli_epi32(m_r_3, 2)); - let m_5r_4 = _mm256_add_epi32(m_r_4, _mm256_slli_epi32(m_r_4, 2)); - - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v0, m_5r_3)); - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, m_5r_4)); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, m_5r_4)); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v2, m_5r_3)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v2, m_5r_4)); - - x.v1 = _mm256_permutevar8x32_epi32(x.v1, swap); - - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v1, m_5r_3)); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, m_5r_4)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v1, m_r_0)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, m_r_1)); - - let m_5r_1 = _mm256_permutevar8x32_epi32(m_5r_4, swap); - let m_5r_2 = _mm256_permutevar8x32_epi32(m_5r_3, swap); - - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, m_5r_2)); - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v2, m_5r_1)); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v2, m_5r_2)); - - Unreduced4x130 { v0, v1, v2, v3, v4 } + // SAFETY: the caller guarantees AVX2 support; the documented bounds keep every 64-bit lane sum below 2^61. + unsafe { + let mut x = x; + let r1 = m.r1.a; + + let v0u = _mm256_unpacklo_epi32(m.v0, m.v1); + let v1u = _mm256_unpackhi_epi32(m.v0, m.v1); + + let ord_a = _mm256_set_epi32(1, 0, 6, 7, 2, 0, 3, 1); + let m_r_0 = _mm256_blend_epi32( + _mm256_permutevar8x32_epi32(r1, ord_a), + _mm256_permutevar8x32_epi32(v0u, ord_a), + 0b00111111, + ); + let ord_b = _mm256_set_epi32(3, 2, 4, 5, 2, 0, 3, 1); + let m_r_2 = _mm256_blend_epi32( + _mm256_permutevar8x32_epi32(r1, ord_b), + _mm256_permutevar8x32_epi32(v1u, ord_b), + 0b00111111, + ); + let ord_c = _mm256_set_epi32(1, 4, 6, 6, 2, 4, 3, 5); + let m_r_4 = _mm256_blend_epi32( + _mm256_blend_epi32( + _mm256_permutevar8x32_epi32(r1, ord_c), + _mm256_permutevar8x32_epi32(v1u, ord_c), + 0b00010000, + ), + _mm256_permutevar8x32_epi32(v0u, ord_c), + 0b00101111, + ); + + let mut v0 = _mm256_mul_epu32(x.v0, m_r_0); + let mut v1 = _mm256_mul_epu32(x.v1, m_r_0); + let mut v2 = _mm256_mul_epu32(x.v0, m_r_2); + let mut v3 = _mm256_mul_epu32(x.v1, m_r_2); + let mut v4 = _mm256_mul_epu32(x.v0, m_r_4); + + let swap = _mm256_set_epi32(6, 7, 4, 5, 2, 3, 0, 1); + let m_r_1 = _mm256_permutevar8x32_epi32(m_r_0, swap); + let m_r_3 = _mm256_permutevar8x32_epi32(m_r_2, swap); + + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, m_r_1)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, m_r_1)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, m_r_3)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, m_r_3)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v2, m_r_0)); + + x.v0 = _mm256_permutevar8x32_epi32(x.v0, swap); + + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v0, m_r_0)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, m_r_1)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v0, m_r_2)); + + let m_5r_3 = _mm256_add_epi32(m_r_3, _mm256_slli_epi32(m_r_3, 2)); + let m_5r_4 = _mm256_add_epi32(m_r_4, _mm256_slli_epi32(m_r_4, 2)); + + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v0, m_5r_3)); + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, m_5r_4)); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, m_5r_4)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v2, m_5r_3)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v2, m_5r_4)); + + x.v1 = _mm256_permutevar8x32_epi32(x.v1, swap); + + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v1, m_5r_3)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, m_5r_4)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v1, m_r_0)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, m_r_1)); + + let m_5r_1 = _mm256_permutevar8x32_epi32(m_5r_4, swap); + let m_5r_2 = _mm256_permutevar8x32_epi32(m_5r_3, swap); + + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, m_5r_2)); + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v2, m_5r_1)); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v2, m_5r_2)); + + Unreduced4x130 { v0, v1, v2, v3, v4 } + } } // ── Unreduced4x130 ────────────────────────────────────────────────────── impl Unreduced4x130 { /// Carry-reduce 4 values in parallel back to 26-bit limbs. + /// + /// `self` must contain bounded convolution coefficients produced by [`mul_4x130`] or [`mul_spaced`]. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn reduce(self) -> Aligned4x130 { - let mask_26 = _mm256_set1_epi64x(0x3ff_ffff); - - let adc = |x1: __m256i, x0: __m256i| -> (__m256i, __m256i) { - let y1 = _mm256_add_epi64(x1, _mm256_srli_epi64(x0, 26)); - let y0 = _mm256_and_si256(x0, mask_26); - (y1, y0) - }; - let red = |x4: __m256i, x0: __m256i| -> (__m256i, __m256i) { - let y0 = _mm256_add_epi64(x0, _mm256_mul_epu32(_mm256_srli_epi64(x4, 26), _mm256_set1_epi64x(5))); - let y4 = _mm256_and_si256(x4, mask_26); - (y4, y0) - }; - - let (r1, r0) = adc(self.v1, self.v0); - let (r4, r3) = adc(self.v4, self.v3); - let (r2, r1) = adc(self.v2, r1); - let (r4, r0) = red(r4, r0); - let (r3, r2) = adc(r3, r2); - let (r1, r0) = adc(r1, r0); - let (r4, r3) = adc(r4, r3); - - Aligned4x130 { - v0: _mm256_blend_epi32(r0, _mm256_slli_epi64(r2, 32), 0b10101010), - v1: _mm256_blend_epi32(r1, _mm256_slli_epi64(r3, 32), 0b10101010), - v2: r4, + // SAFETY: the caller guarantees AVX2 support. The bounded coefficients keep every carry, fivefold fold, and lane + // sum in u64; both closures and the final packing are register-only. + unsafe { + let mask_26 = _mm256_set1_epi64x(0x3ff_ffff); + + let adc = |x1: __m256i, x0: __m256i| -> (__m256i, __m256i) { + let y1 = _mm256_add_epi64(x1, _mm256_srli_epi64(x0, 26)); + let y0 = _mm256_and_si256(x0, mask_26); + (y1, y0) + }; + let red = |x4: __m256i, x0: __m256i| -> (__m256i, __m256i) { + let y0 = _mm256_add_epi64(x0, _mm256_mul_epu32(_mm256_srli_epi64(x4, 26), _mm256_set1_epi64x(5))); + let y4 = _mm256_and_si256(x4, mask_26); + (y4, y0) + }; + + let (r1, r0) = adc(self.v1, self.v0); + let (r4, r3) = adc(self.v4, self.v3); + let (r2, r1) = adc(self.v2, r1); + let (r4, r0) = red(r4, r0); + let (r3, r2) = adc(r3, r2); + let (r1, r0) = adc(r1, r0); + let (r4, r3) = adc(r4, r3); + + Aligned4x130 { + v0: _mm256_blend_epi32(r0, _mm256_slli_epi64(r2, 32), 0b10101010), + v1: _mm256_blend_epi32(r1, _mm256_slli_epi64(r3, 32), 0b10101010), + v2: r4, + } } } /// Horizontal sum of 4 lanes into a single `Unreduced130`. + /// + /// `self` must contain the bounded four-stream convolution produced by [`mul_spaced`]. The resulting coefficient + /// sums remain below 2⁶³. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn sum(self) -> Unreduced130 { - let lo01 = _mm256_add_epi64( - _mm256_unpackhi_epi64(self.v0, self.v1), - _mm256_unpacklo_epi64(self.v0, self.v1), - ); - let lo23 = _mm256_add_epi64( - _mm256_unpackhi_epi64(self.v2, self.v3), - _mm256_unpacklo_epi64(self.v2, self.v3), - ); - let v0 = _mm256_add_epi64( - _mm256_inserti128_si256(lo01, _mm256_castsi256_si128(lo23), 1), - _mm256_inserti128_si256(lo23, _mm256_extracti128_si256(lo01, 1), 0), - ); - let v4 = _mm256_add_epi64(self.v4, _mm256_permute4x64_epi64(self.v4, imm8(1, 0, 3, 2))); - let v1 = _mm256_add_epi64(v4, _mm256_permute4x64_epi64(v4, imm8(0, 0, 0, 1))); - Unreduced130 { v0, v1 } + // SAFETY: the caller guarantees AVX2 support; the documented bounds keep every horizontal lane sum in u64. + unsafe { + let lo01 = _mm256_add_epi64( + _mm256_unpackhi_epi64(self.v0, self.v1), + _mm256_unpacklo_epi64(self.v0, self.v1), + ); + let lo23 = _mm256_add_epi64( + _mm256_unpackhi_epi64(self.v2, self.v3), + _mm256_unpacklo_epi64(self.v2, self.v3), + ); + let v0 = _mm256_add_epi64( + _mm256_inserti128_si256(lo01, _mm256_castsi256_si128(lo23), 1), + _mm256_inserti128_si256(lo23, _mm256_extracti128_si256(lo01, 1), 0), + ); + let v4 = _mm256_add_epi64(self.v4, _mm256_permute4x64_epi64(self.v4, imm8(1, 0, 3, 2))); + let v1 = _mm256_add_epi64(v4, _mm256_permute4x64_epi64(v4, imm8(0, 0, 0, 1))); + Unreduced130 { v0, v1 } + } } } @@ -478,24 +628,34 @@ impl Unreduced4x130 { impl SpacedMultiplier4x130 { /// Compute `(multiplier, R⁴)` from `(R¹, R²)`. + /// + /// `r1` and `r2` must be reduced consecutive powers of the same clamped Poly1305 key. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn new(r1: PrecomputedMultiplier, r2: PrecomputedMultiplier) -> (Self, PrecomputedMultiplier) { - let r3 = mul_single(Aligned130(r2.a), r1).reduce(); - let r4 = mul_single(Aligned130(r2.a), r2).reduce(); - - let v0 = _mm256_blend_epi32( - r3.0, - _mm256_permutevar8x32_epi32(r2.a, _mm256_set_epi32(4, 3, 1, 0, 0, 0, 0, 0)), - 0b11100000, - ); - let v1 = _mm256_blend_epi32( - r4.0, - _mm256_permutevar8x32_epi32(r2.a, _mm256_set_epi32(4, 2, 0, 0, 0, 0, 0, 0)), - 0b11100000, - ); - - let m = SpacedMultiplier4x130 { v0, v1, r1 }; - (m, PrecomputedMultiplier::from_aligned(r4)) + // SAFETY: the caller guarantees AVX2 support. Matching bounded powers satisfy both nested multiplication/reduction + // contracts; the remaining operations only pack those register values. + unsafe { + let r3 = mul_single(Aligned130(r2.a), r1).reduce(); + let r4 = mul_single(Aligned130(r2.a), r2).reduce(); + + let v0 = _mm256_blend_epi32( + r3.0, + _mm256_permutevar8x32_epi32(r2.a, _mm256_set_epi32(4, 3, 1, 0, 0, 0, 0, 0)), + 0b11100000, + ); + let v1 = _mm256_blend_epi32( + r4.0, + _mm256_permutevar8x32_epi32(r2.a, _mm256_set_epi32(4, 2, 0, 0, 0, 0, 0, 0)), + 0b11100000, + ); + + let m = SpacedMultiplier4x130 { v0, v1, r1 }; + (m, PrecomputedMultiplier::from_aligned(r4)) + } } } @@ -509,21 +669,16 @@ struct Par4State { r4: PrecomputedMultiplier, } -/// Authenticate `(aad, ciphertext)` using 4-way parallel Poly1305. +/// Authenticate `(aad, ciphertext)` using the AVX2 4-way Poly1305 kernel. /// /// Uses its own AVX2 kernel — ignores the per-block `ComputeBlockFn` dispatch. -pub(super) fn authenticate_aead_par4( - aad: &[u8], - ciphertext: &[u8], - key: &[u8; 32], - lengths: AeadByteLengths, -) -> [u8; 16] { - // SAFETY: caller verified AVX2 capability via `current_caps().has(x86::AVX2)`. - unsafe { authenticate_aead_par4_avx2(aad, ciphertext, key, lengths) } -} - +/// `lengths` must encode the lengths of `aad` and `ciphertext`. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available. #[target_feature(enable = "avx2")] -unsafe fn authenticate_aead_par4_avx2( +pub(super) unsafe fn authenticate_aead_par4( aad: &[u8], ciphertext: &[u8], key: &[u8; 32], @@ -532,9 +687,14 @@ unsafe fn authenticate_aead_par4_avx2( let state = State::new(key); // Precompute R¹, R² as AVX2 multipliers. - let r = Aligned130::from_limbs(state.r); - let r1 = PrecomputedMultiplier::from_aligned(r); - let r2 = PrecomputedMultiplier::from_aligned(mul_single(Aligned130(r1.a), r1).reduce()); + // SAFETY: the caller guarantees AVX2. `State::new` produces a clamped multiplier, and each nested helper preserves + // the bounded representation while deriving consecutive powers from that same key. + let (r1, r2) = unsafe { + let r = Aligned130::from_limbs(state.r); + let r1 = PrecomputedMultiplier::from_aligned(r); + let r2 = PrecomputedMultiplier::from_aligned(mul_single(Aligned130(r1.a), r1).reduce()); + (r1, r2) + }; // 4-block accumulator (initialized on first 4-block group). let mut acc: Option = None; @@ -547,56 +707,75 @@ unsafe fn authenticate_aead_par4_avx2( for chunk in &mut chunks { let mut block = [0u8; 16]; block.copy_from_slice(chunk); - num_cached = push_block(block, &mut cached, num_cached, &mut acc, r1, r2); + // SAFETY: the caller guarantees AVX2. The loop produces consecutive full blocks; `push_block` preserves + // `num_cached < 4` and keeps the cache, powers, and accumulator in the same key stream. + num_cached = unsafe { push_block(block, &mut cached, num_cached, &mut acc, r1, r2) }; } let rem = chunks.remainder(); if !rem.is_empty() { let mut block = [0u8; 16]; block[..rem.len()].copy_from_slice(rem); - num_cached = push_block(block, &mut cached, num_cached, &mut acc, r1, r2); + // SAFETY: the caller guarantees AVX2. Zero padding completes the segment's final full AEAD block, and the + // established cache and matching-key invariants still hold. + num_cached = unsafe { push_block(block, &mut cached, num_cached, &mut acc, r1, r2) }; } } // Process lengths block. let length_block = lengths.to_le_bytes_block(); - num_cached = push_block(length_block, &mut cached, num_cached, &mut acc, r1, r2); + // SAFETY: the caller guarantees AVX2. The length encoding is the final full AEAD block, and the established cache + // and matching-key invariants still hold. + num_cached = unsafe { push_block(length_block, &mut cached, num_cached, &mut acc, r1, r2) }; // Finalize: merge 4 lanes, process remaining blocks. - let mut p: Option = acc.map(|s| mul_spaced(s.poly, s.spaced).sum().reduce()); + let mut p: Option = acc.map(|s| { + // SAFETY: the caller guarantees AVX2, and `accumulate_4_blocks` constructed `poly` and `spaced` from matching + // bounded powers in this key stream. + unsafe { mul_spaced(s.poly, s.spaced).sum().reduce() } + }); // 2-block tail. if num_cached >= 2 { - let mut c0 = Aligned130::from_block(&cached[0]); - let c1 = Aligned130::from_block(&cached[1]); - if let Some(pv) = p { - c0 = c0.add(pv); - } - let a = mul_single(c0, r2); - let b = mul_single(c1, r1); - p = Some( - Unreduced130 { - v0: _mm256_add_epi64(a.v0, b.v0), - v1: _mm256_add_epi64(a.v1, b.v1), + // SAFETY: the caller guarantees AVX2. The cache invariant proves slots 0 and 1 are consecutive full blocks; + // `p`, `r2`, and `r1` are bounded values from this key stream, and the two products preserve block order. + unsafe { + let mut c0 = Aligned130::from_block(&cached[0]); + let c1 = Aligned130::from_block(&cached[1]); + if let Some(pv) = p { + c0 = c0.add(pv); } - .reduce(), - ); - cached[0] = cached[2]; - num_cached = num_cached.strict_sub(2); + let a = mul_single(c0, r2); + let b = mul_single(c1, r1); + p = Some( + Unreduced130 { + v0: _mm256_add_epi64(a.v0, b.v0), + v1: _mm256_add_epi64(a.v1, b.v1), + } + .reduce(), + ); + cached[0] = cached[2]; + num_cached = num_cached.strict_sub(2); + } } // 1-block tail. if num_cached == 1 { - let mut c = Aligned130::from_block(&cached[0]); - if let Some(pv) = p { - c = c.add(pv); + // SAFETY: the caller guarantees AVX2. The cache invariant proves slot 0 is the final full block, while `p` and + // `r1` are bounded values from the same key stream. + unsafe { + let mut c = Aligned130::from_block(&cached[0]); + if let Some(pv) = p { + c = c.add(pv); + } + p = Some(mul_single(c, r1).reduce()); } - p = Some(mul_single(c, r1).reduce()); } // Convert AVX2 result back to scalar and finalize. let mut final_state = state; if let Some(pv) = p { - final_state.h = pv.into_limbs(); + // SAFETY: the caller guarantees AVX2, and `pv` is the bounded result of this kernel's reduction chain. + final_state.h = unsafe { pv.into_limbs() }; } let tag = final_state.finalize(); ct::zeroize(cached.as_flattened_mut()); @@ -604,6 +783,11 @@ unsafe fn authenticate_aead_par4_avx2( } /// Cache one block; flush a 4-block group when full. Returns updated `num_cached`. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available, `num_cached < 4`, the occupied cache prefix and `block` are +/// consecutive full AEAD-padded blocks, and `r1`, `r2`, and any accumulator belong to the same bounded key stream. #[inline(always)] unsafe fn push_block( block: [u8; 16], @@ -616,7 +800,9 @@ unsafe fn push_block( cached[num_cached] = block; let n = num_cached.strict_add(1); if n == 4 { - accumulate_4_blocks(cached, acc, r1, r2); + // SAFETY: the caller guarantees AVX2 and a matching cache, multiplier, and accumulator. `n == 4` proves that + // this write completed the four-block group before it is accumulated. + unsafe { accumulate_4_blocks(cached, acc, r1, r2) }; 0 } else { n @@ -624,6 +810,11 @@ unsafe fn push_block( } /// Process a full 4-block group into the parallel accumulator. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available, `cached` contains four consecutive full AEAD-padded blocks, and +/// `r1`, `r2`, and any existing accumulator are bounded powers and state derived from the same clamped Poly1305 key. #[inline(always)] unsafe fn accumulate_4_blocks( cached: &[[u8; 16]; 4], @@ -631,15 +822,19 @@ unsafe fn accumulate_4_blocks( r1: PrecomputedMultiplier, r2: PrecomputedMultiplier, ) { - let blocks = Aligned4x130::from_blocks(cached); - if let Some(ref mut s) = *acc { - s.poly = mul_4x130(&s.poly, s.r4).reduce().add(blocks); - } else { - let (spaced, r4) = SpacedMultiplier4x130::new(r1, r2); - *acc = Some(Par4State { - poly: blocks, - spaced, - r4, - }); + // SAFETY: the caller guarantees AVX2 and the matching-key block, multiplier, and accumulator invariants required + // by each nested helper. All operations are fixed-width loads or register-only arithmetic. + unsafe { + let blocks = Aligned4x130::from_blocks(cached); + if let Some(ref mut s) = *acc { + s.poly = mul_4x130(&s.poly, s.r4).reduce().add(blocks); + } else { + let (spaced, r4) = SpacedMultiplier4x130::new(r1, r2); + *acc = Some(Par4State { + poly: blocks, + spaced, + r4, + }); + } } } diff --git a/src/aead/polyval.rs b/src/aead/polyval.rs index f6f65c08..c5c1db61 100644 --- a/src/aead/polyval.rs +++ b/src/aead/polyval.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! Fixed-schedule, table-free POLYVAL universal hash (RFC 8452). //! //! Generated-code timing claims remain configuration- and @@ -31,6 +29,32 @@ pub(crate) const KEY_SIZE: usize = 16; #[cfg(test)] const POLY: u128 = (1u128 << 127) | (1u128 << 126) | (1u128 << 121) | 1; +#[inline(always)] +fn split_u128_le(value: u128) -> (u64, u64) { + let [ + lo_0, + lo_1, + lo_2, + lo_3, + lo_4, + lo_5, + lo_6, + lo_7, + hi_0, + hi_1, + hi_2, + hi_3, + hi_4, + hi_5, + hi_6, + hi_7, + ] = value.to_le_bytes(); + ( + u64::from_le_bytes([lo_0, lo_1, lo_2, lo_3, lo_4, lo_5, lo_6, lo_7]), + u64::from_le_bytes([hi_0, hi_1, hi_2, hi_3, hi_4, hi_5, hi_6, hi_7]), + ) +} + // x86_64 PCLMULQDQ backend #[cfg(target_arch = "x86_64")] @@ -47,8 +71,8 @@ mod pclmul { pub(super) unsafe fn clmul128_reduce(a: u128, b: u128) -> u128 { // SAFETY: target_feature gate guarantees PCLMULQDQ + SSE2. unsafe { - let a_xmm = _mm_loadu_si128((&a as *const u128).cast()); - let b_xmm = _mm_loadu_si128((&b as *const u128).cast()); + let a_xmm = _mm_loadu_si128(core::ptr::from_ref(&a).cast()); + let b_xmm = _mm_loadu_si128(core::ptr::from_ref(&b).cast()); // Schoolbook 128×128 → 256-bit product (4 PCLMULQDQ instructions). let lo = _mm_clmulepi64_si128(a_xmm, b_xmm, 0x00); // a_lo × b_lo @@ -65,7 +89,7 @@ mod pclmul { let result = mont_reduce_sse2(lo_128, hi_128); let mut out = 0u128; - _mm_storeu_si128((&mut out as *mut u128).cast(), result); + _mm_storeu_si128(core::ptr::from_mut(&mut out).cast(), result); out } } @@ -83,14 +107,14 @@ mod pclmul { // fixed-size stack/reference values. unsafe { let b0 = acc ^ blocks[0]; - let d0 = _mm_loadu_si128((&b0 as *const u128).cast()); - let d1 = _mm_loadu_si128((&blocks[1] as *const u128).cast()); - let d2 = _mm_loadu_si128((&blocks[2] as *const u128).cast()); - let d3 = _mm_loadu_si128((&blocks[3] as *const u128).cast()); - let h0 = _mm_loadu_si128((&h_powers_rev[0] as *const u128).cast()); - let h1 = _mm_loadu_si128((&h_powers_rev[1] as *const u128).cast()); - let h2 = _mm_loadu_si128((&h_powers_rev[2] as *const u128).cast()); - let h3 = _mm_loadu_si128((&h_powers_rev[3] as *const u128).cast()); + let d0 = _mm_loadu_si128(core::ptr::from_ref(&b0).cast()); + let d1 = _mm_loadu_si128(core::ptr::from_ref(&blocks[1]).cast()); + let d2 = _mm_loadu_si128(core::ptr::from_ref(&blocks[2]).cast()); + let d3 = _mm_loadu_si128(core::ptr::from_ref(&blocks[3]).cast()); + let h0 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[0]).cast()); + let h1 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[1]).cast()); + let h2 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[2]).cast()); + let h3 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[3]).cast()); aggregate_xmms([d0, d1, d2, d3], [h0, h1, h2, h3]) } @@ -118,20 +142,22 @@ mod pclmul { // fixed-size references, and PSHUFB only shuffles bytes within each register. unsafe { let reverse_bytes = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let acc_xmm = _mm_loadu_si128((&acc as *const u128).cast()); + let acc_xmm = _mm_loadu_si128(core::ptr::from_ref(&acc).cast()); let d0 = _mm_xor_si128(_mm_shuffle_epi8(raw0, reverse_bytes), acc_xmm); let d1 = _mm_shuffle_epi8(raw1, reverse_bytes); let d2 = _mm_shuffle_epi8(raw2, reverse_bytes); let d3 = _mm_shuffle_epi8(raw3, reverse_bytes); - let h0 = _mm_loadu_si128((&h_powers_rev[0] as *const u128).cast()); - let h1 = _mm_loadu_si128((&h_powers_rev[1] as *const u128).cast()); - let h2 = _mm_loadu_si128((&h_powers_rev[2] as *const u128).cast()); - let h3 = _mm_loadu_si128((&h_powers_rev[3] as *const u128).cast()); + let h0 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[0]).cast()); + let h1 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[1]).cast()); + let h2 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[2]).cast()); + let h3 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[3]).cast()); aggregate_xmms([d0, d1, d2, d3], [h0, h1, h2, h3]) } } + /// # Safety + /// Caller must ensure PCLMULQDQ and SSE2 are available. #[target_feature(enable = "pclmulqdq,sse2")] #[inline] unsafe fn aggregate_xmms(data: [__m128i; 4], h: [__m128i; 4]) -> u128 { @@ -159,7 +185,7 @@ mod pclmul { let result = mont_reduce_sse2(lo_sum, hi_sum); let mut out = 0u128; - _mm_storeu_si128((&mut out as *mut u128).cast(), result); + _mm_storeu_si128(core::ptr::from_mut(&mut out).cast(), result); out } } @@ -168,7 +194,7 @@ mod pclmul { /// /// Equivalent to the portable `mont_reduce` but uses SSE2 lane-parallel shifts. #[inline] - pub(super) unsafe fn mont_reduce_sse2(lo: __m128i, hi: __m128i) -> __m128i { + pub(super) fn mont_reduce_sse2(lo: __m128i, hi: __m128i) -> __m128i { // SAFETY: caller guarantees SSE2 availability via target_feature chain. unsafe { // Phase 1: Compute left-shift contribution from both lo lanes. @@ -204,6 +230,7 @@ mod pclmul { #[cfg(target_arch = "aarch64")] mod pmull { + use super::split_u128_le; use core::arch::aarch64::*; /// Core PMULL multiply + reduce — `#[inline(always)]` for guaranteed inlining. @@ -212,15 +239,18 @@ mod pmull { /// features on the immediate caller. Combined with `#[inline(always)]`, /// the function body is inlined into the caller — no function call boundary, /// no register spills. + /// + /// # Safety + /// + /// The current CPU must support NEON and PMULL. The caller must enter through + /// an `aes,neon` target-feature scope selected by validated runtime capabilities. #[target_feature(enable = "neon", enable = "aes")] #[inline] pub(super) unsafe fn clmul128_reduce_core(a: u128, b: u128) -> u128 { // SAFETY: caller guarantees NEON + PMULL via target_feature chain. unsafe { - let a_lo = a as u64; - let a_hi = (a >> 64) as u64; - let b_lo = b as u64; - let b_hi = (b >> 64) as u64; + let (a_lo, a_hi) = split_u128_le(a); + let (b_lo, b_hi) = split_u128_le(b); // Karatsuba 128×128 → 256-bit product (3 PMULL instructions). let ll = vreinterpretq_u64_p128(vmull_p64(a_lo, b_lo)); @@ -266,6 +296,11 @@ mod pmull { /// Uses NEON lane-parallel shifts (`vshlq_n_u64`, `vshrq_n_u64`) and /// `vextq_u64` for cross-lane propagation — structurally identical to /// the SSE2 `mont_reduce_sse2` path. + /// + /// # Safety + /// + /// The caller must execute in a NEON target-feature scope. Every bit pattern + /// in `lo` and `hi` is a valid unreduced field product. #[inline] unsafe fn mont_reduce_neon(lo: uint64x2_t, hi: uint64x2_t) -> uint64x2_t { // SAFETY: caller guarantees NEON availability via target_feature chain. @@ -300,6 +335,11 @@ mod pmull { /// Computes `(acc ^ b0) * H^4 ^ b1 * H^3 ^ b2 * H^2 ^ b3 * H` using /// 12 independent `vmull_p64` instructions that the OOO core on Neoverse /// V1/V2 (2 crypto pipes) can schedule freely, then a single reduction. + /// + /// # Safety + /// + /// The current CPU must support NEON and PMULL. The caller must enter through + /// an `aes,neon` target-feature scope selected by validated runtime capabilities. #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] #[target_feature(enable = "neon", enable = "aes")] #[inline] @@ -312,34 +352,26 @@ mod pmull { let b3 = blocks[3]; // 12 vmull_p64: 4 blocks × 3 Karatsuba products. - let b0_lo = b0 as u64; - let b0_hi = (b0 >> 64) as u64; - let h0_lo = h_powers_rev[0] as u64; - let h0_hi = (h_powers_rev[0] >> 64) as u64; + let (b0_lo, b0_hi) = split_u128_le(b0); + let (h0_lo, h0_hi) = split_u128_le(h_powers_rev[0]); let ll0 = vreinterpretq_u64_p128(vmull_p64(b0_lo, h0_lo)); let hh0 = vreinterpretq_u64_p128(vmull_p64(b0_hi, h0_hi)); let mm0 = vreinterpretq_u64_p128(vmull_p64(b0_lo ^ b0_hi, h0_lo ^ h0_hi)); - let b1_lo = b1 as u64; - let b1_hi = (b1 >> 64) as u64; - let h1_lo = h_powers_rev[1] as u64; - let h1_hi = (h_powers_rev[1] >> 64) as u64; + let (b1_lo, b1_hi) = split_u128_le(b1); + let (h1_lo, h1_hi) = split_u128_le(h_powers_rev[1]); let ll1 = vreinterpretq_u64_p128(vmull_p64(b1_lo, h1_lo)); let hh1 = vreinterpretq_u64_p128(vmull_p64(b1_hi, h1_hi)); let mm1 = vreinterpretq_u64_p128(vmull_p64(b1_lo ^ b1_hi, h1_lo ^ h1_hi)); - let b2_lo = b2 as u64; - let b2_hi = (b2 >> 64) as u64; - let h2_lo = h_powers_rev[2] as u64; - let h2_hi = (h_powers_rev[2] >> 64) as u64; + let (b2_lo, b2_hi) = split_u128_le(b2); + let (h2_lo, h2_hi) = split_u128_le(h_powers_rev[2]); let ll2 = vreinterpretq_u64_p128(vmull_p64(b2_lo, h2_lo)); let hh2 = vreinterpretq_u64_p128(vmull_p64(b2_hi, h2_hi)); let mm2 = vreinterpretq_u64_p128(vmull_p64(b2_lo ^ b2_hi, h2_lo ^ h2_hi)); - let b3_lo = b3 as u64; - let b3_hi = (b3 >> 64) as u64; - let h3_lo = h_powers_rev[3] as u64; - let h3_hi = (h_powers_rev[3] >> 64) as u64; + let (b3_lo, b3_hi) = split_u128_le(b3); + let (h3_lo, h3_hi) = split_u128_le(h_powers_rev[3]); let ll3 = vreinterpretq_u64_p128(vmull_p64(b3_lo, h3_lo)); let hh3 = vreinterpretq_u64_p128(vmull_p64(b3_hi, h3_hi)); let mm3 = vreinterpretq_u64_p128(vmull_p64(b3_lo ^ b3_hi, h3_lo ^ h3_hi)); @@ -363,6 +395,11 @@ mod pmull { } /// 8-block GHASH aggregate from big-endian ciphertext lanes already held in NEON registers. + /// + /// # Safety + /// + /// The current CPU must support NEON and PMULL. The caller must enter through + /// an `aes,neon` target-feature scope selected by validated runtime capabilities. #[cfg(feature = "aes-gcm")] #[target_feature(enable = "neon", enable = "aes")] #[inline] @@ -375,6 +412,12 @@ mod pmull { // GHASH lanes to the little-endian POLYVAL-domain lane representation. unsafe { #[inline(always)] + /// Loads one little-endian field element into two NEON lanes. + /// + /// # Safety + /// + /// `power` must be valid to read one initialized `u128`. The caller must + /// execute in a NEON target-feature scope. unsafe fn load_power(power: *const u128) -> uint64x2_t { // SAFETY: caller passes a pointer into `h_powers_rev`; `vld1q_u64` // accepts arbitrary alignment and reads exactly one initialized u128. @@ -382,10 +425,16 @@ mod pmull { } #[inline(always)] + /// Places one little-endian field element into two NEON lanes. + /// + /// # Safety + /// + /// The caller must execute in a NEON target-feature scope. unsafe fn u128_to_lanes(x: u128) -> uint64x2_t { + let (lo, hi) = split_u128_le(x); // SAFETY: caller is inside a NEON target scope. `vcreate_u64` // initializes one 64-bit lane and `vcombine_u64` builds the pair. - unsafe { vcombine_u64(vcreate_u64(x as u64), vcreate_u64((x >> 64) as u64)) } + unsafe { vcombine_u64(vcreate_u64(lo), vcreate_u64(hi)) } } let mut ll = vdupq_n_u64(0); @@ -447,7 +496,6 @@ mod pmull { // s390x VGFM backend (Galois field multiply) #[cfg(target_arch = "s390x")] -#[allow(unsafe_code)] mod s390x_vgfm { use core::{arch::asm, simd::i64x2}; @@ -455,11 +503,14 @@ mod s390x_vgfm { /// /// Places operands in the low lane (element 1) with the high lane zeroed, /// so VGFM computes `0*0 XOR a*b = a*b`. + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn mul64(a: u64, b: u64) -> i64x2 { - let va = i64x2::from_array([0, a as i64]); - let vb = i64x2::from_array([0, b as i64]); + let va = i64x2::from_array([0, a.cast_signed()]); + let vb = i64x2::from_array([0, b.cast_signed()]); // SAFETY: Caller guarantees z/Vector facility is available. unsafe { let out: i64x2; @@ -475,6 +526,9 @@ mod s390x_vgfm { } /// Per-lane left shift of both 64-bit elements. + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn veslg(a: i64x2) -> i64x2 { @@ -493,6 +547,9 @@ mod s390x_vgfm { } /// Per-lane logical right shift of both 64-bit elements. + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vesrlg(a: i64x2) -> i64x2 { @@ -514,6 +571,9 @@ mod s390x_vgfm { /// /// - `vsldb(v, zero, 8)`: moves low lane to high, zeros low = `v << 64` /// - `vsldb(zero, v, 8)`: moves high lane to low, zeros high = `v >> 64` + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vsldb(a: i64x2, b: i64x2) -> i64x2 { @@ -537,6 +597,9 @@ mod s390x_vgfm { /// Structurally identical to `mont_reduce_sse2` / `mont_reduce_neon`, /// using s390x VESLG/VESRLG for per-lane shifts and VSLDB for cross-lane /// byte shifts. + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn mont_reduce(lo: i64x2, hi: i64x2) -> i64x2 { @@ -564,15 +627,16 @@ mod s390x_vgfm { } /// Core VGFM multiply + reduce — `#[inline(always)]` for guaranteed inlining. + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[target_feature(enable = "vector")] #[inline] pub(super) unsafe fn clmul128_reduce_core(a: u128, b: u128) -> u128 { // SAFETY: caller guarantees z/Vector availability via target_feature chain. unsafe { - let a_lo = a as u64; - let a_hi = (a >> 64) as u64; - let b_lo = b as u64; - let b_hi = (b >> 64) as u64; + let (a_lo, a_hi) = super::split_u128_le(a); + let (b_lo, b_hi) = super::split_u128_le(b); let zero = i64x2::from_array([0, 0]); // Karatsuba: 3 VGFM multiplies. @@ -587,7 +651,7 @@ mod s390x_vgfm { let result = mont_reduce(lo_128, hi_128); let arr = result.to_array(); - ((arr[0] as u64 as u128) << 64) | (arr[1] as u64 as u128) + (u128::from(arr[0].cast_unsigned()) << 64) | u128::from(arr[1].cast_unsigned()) } } @@ -607,6 +671,9 @@ mod s390x_vgfm { /// /// 12 independent VGFM (4 × 3 Karatsuba), then one vector /// Montgomery reduction. + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] #[target_feature(enable = "vector")] #[inline] @@ -617,36 +684,32 @@ mod s390x_vgfm { let b1 = blocks[1]; let b2 = blocks[2]; let b3 = blocks[3]; + let (b0_lo, b0_hi) = super::split_u128_le(b0); + let (b1_lo, b1_hi) = super::split_u128_le(b1); + let (b2_lo, b2_hi) = super::split_u128_le(b2); + let (b3_lo, b3_hi) = super::split_u128_le(b3); + let (h0_lo, h0_hi) = super::split_u128_le(h_powers_rev[0]); + let (h1_lo, h1_hi) = super::split_u128_le(h_powers_rev[1]); + let (h2_lo, h2_hi) = super::split_u128_le(h_powers_rev[2]); + let (h3_lo, h3_hi) = super::split_u128_le(h_powers_rev[3]); let zero = i64x2::from_array([0, 0]); // 12 VGFM: 4 blocks × 3 Karatsuba multiplies. - let v0_0 = mul64(b0 as u64, h_powers_rev[0] as u64); - let v1_0 = mul64((b0 >> 64) as u64, (h_powers_rev[0] >> 64) as u64); - let v2_0 = mul64( - b0 as u64 ^ (b0 >> 64) as u64, - h_powers_rev[0] as u64 ^ (h_powers_rev[0] >> 64) as u64, - ); + let v0_0 = mul64(b0_lo, h0_lo); + let v1_0 = mul64(b0_hi, h0_hi); + let v2_0 = mul64(b0_lo ^ b0_hi, h0_lo ^ h0_hi); - let v0_1 = mul64(b1 as u64, h_powers_rev[1] as u64); - let v1_1 = mul64((b1 >> 64) as u64, (h_powers_rev[1] >> 64) as u64); - let v2_1 = mul64( - b1 as u64 ^ (b1 >> 64) as u64, - h_powers_rev[1] as u64 ^ (h_powers_rev[1] >> 64) as u64, - ); + let v0_1 = mul64(b1_lo, h1_lo); + let v1_1 = mul64(b1_hi, h1_hi); + let v2_1 = mul64(b1_lo ^ b1_hi, h1_lo ^ h1_hi); - let v0_2 = mul64(b2 as u64, h_powers_rev[2] as u64); - let v1_2 = mul64((b2 >> 64) as u64, (h_powers_rev[2] >> 64) as u64); - let v2_2 = mul64( - b2 as u64 ^ (b2 >> 64) as u64, - h_powers_rev[2] as u64 ^ (h_powers_rev[2] >> 64) as u64, - ); + let v0_2 = mul64(b2_lo, h2_lo); + let v1_2 = mul64(b2_hi, h2_hi); + let v2_2 = mul64(b2_lo ^ b2_hi, h2_lo ^ h2_hi); - let v0_3 = mul64(b3 as u64, h_powers_rev[3] as u64); - let v1_3 = mul64((b3 >> 64) as u64, (h_powers_rev[3] >> 64) as u64); - let v2_3 = mul64( - b3 as u64 ^ (b3 >> 64) as u64, - h_powers_rev[3] as u64 ^ (h_powers_rev[3] >> 64) as u64, - ); + let v0_3 = mul64(b3_lo, h3_lo); + let v1_3 = mul64(b3_hi, h3_hi); + let v2_3 = mul64(b3_lo ^ b3_hi, h3_lo ^ h3_hi); // XOR all 4 Karatsuba intermediates (i64x2 vector XOR). let v0 = v0_0 ^ v0_1 ^ v0_2 ^ v0_3; @@ -659,7 +722,7 @@ mod s390x_vgfm { let result = mont_reduce(lo_128, hi_128); let arr = result.to_array(); - ((arr[0] as u64 as u128) << 64) | (arr[1] as u64 as u128) + (u128::from(arr[0].cast_unsigned()) << 64) | u128::from(arr[1].cast_unsigned()) } } } @@ -667,7 +730,6 @@ mod s390x_vgfm { // powerpc64 VPMSUMD backend (polynomial multiply-sum doubleword) #[cfg(target_arch = "powerpc64")] -#[allow(unsafe_code)] mod ppc_vpmsum { use core::{arch::asm, simd::i64x2}; @@ -675,11 +737,14 @@ mod ppc_vpmsum { /// /// Match the existing POWER checksum kernels' lane convention: /// lane 0 carries the low 64 bits, lane 1 the high 64 bits. + /// + /// # Safety + /// Caller must ensure POWER8 vector crypto is available. #[inline] #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn mul64(a: u64, b: u64) -> (u64, u64) { - let va = i64x2::from_array([a as i64, 0]); - let vb = i64x2::from_array([b as i64, 0]); + let va = i64x2::from_array([a.cast_signed(), 0]); + let vb = i64x2::from_array([b.cast_signed(), 0]); // SAFETY: Caller guarantees POWER8 crypto availability. unsafe { let out: i64x2; @@ -691,20 +756,21 @@ mod ppc_vpmsum { options(nomem, nostack, pure), ); let [lo, hi] = out.to_array(); - (lo as u64, hi as u64) + (lo.cast_unsigned(), hi.cast_unsigned()) } } /// Core VPMSUMD multiply + reduce — `#[inline(always)]` for guaranteed inlining. + /// + /// # Safety + /// Caller must ensure POWER8 vector crypto is available. #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] #[inline] pub(super) unsafe fn clmul128_reduce_core(a: u128, b: u128) -> u128 { // SAFETY: caller guarantees POWER8 crypto availability via target_feature chain. unsafe { - let a_lo = a as u64; - let a_hi = (a >> 64) as u64; - let b_lo = b as u64; - let b_hi = (b >> 64) as u64; + let (a_lo, a_hi) = super::split_u128_le(a); + let (b_lo, b_hi) = super::split_u128_le(b); // Karatsuba: 3 vpmsumd multiplies. let (v0_lo, v0_hi) = mul64(a_lo, b_lo); @@ -735,6 +801,9 @@ mod ppc_vpmsum { /// /// 12 independent `vpmsumd` (4 × 3 Karatsuba), then one scalar /// Montgomery reduction. + /// + /// # Safety + /// Caller must ensure POWER8 vector crypto is available. #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] #[inline] @@ -745,35 +814,31 @@ mod ppc_vpmsum { let b1 = blocks[1]; let b2 = blocks[2]; let b3 = blocks[3]; + let (b0_lo, b0_hi) = super::split_u128_le(b0); + let (b1_lo, b1_hi) = super::split_u128_le(b1); + let (b2_lo, b2_hi) = super::split_u128_le(b2); + let (b3_lo, b3_hi) = super::split_u128_le(b3); + let (h0_lo, h0_hi) = super::split_u128_le(h_powers_rev[0]); + let (h1_lo, h1_hi) = super::split_u128_le(h_powers_rev[1]); + let (h2_lo, h2_hi) = super::split_u128_le(h_powers_rev[2]); + let (h3_lo, h3_hi) = super::split_u128_le(h_powers_rev[3]); // 12 vpmsumd: 4 blocks × 3 Karatsuba multiplies. - let (z0_0l, z0_0h) = mul64(b0 as u64, h_powers_rev[0] as u64); - let (z1_0l, z1_0h) = mul64((b0 >> 64) as u64, (h_powers_rev[0] >> 64) as u64); - let (z2_0l, z2_0h) = mul64( - b0 as u64 ^ (b0 >> 64) as u64, - h_powers_rev[0] as u64 ^ (h_powers_rev[0] >> 64) as u64, - ); + let (z0_0l, z0_0h) = mul64(b0_lo, h0_lo); + let (z1_0l, z1_0h) = mul64(b0_hi, h0_hi); + let (z2_0l, z2_0h) = mul64(b0_lo ^ b0_hi, h0_lo ^ h0_hi); - let (z0_1l, z0_1h) = mul64(b1 as u64, h_powers_rev[1] as u64); - let (z1_1l, z1_1h) = mul64((b1 >> 64) as u64, (h_powers_rev[1] >> 64) as u64); - let (z2_1l, z2_1h) = mul64( - b1 as u64 ^ (b1 >> 64) as u64, - h_powers_rev[1] as u64 ^ (h_powers_rev[1] >> 64) as u64, - ); + let (z0_1l, z0_1h) = mul64(b1_lo, h1_lo); + let (z1_1l, z1_1h) = mul64(b1_hi, h1_hi); + let (z2_1l, z2_1h) = mul64(b1_lo ^ b1_hi, h1_lo ^ h1_hi); - let (z0_2l, z0_2h) = mul64(b2 as u64, h_powers_rev[2] as u64); - let (z1_2l, z1_2h) = mul64((b2 >> 64) as u64, (h_powers_rev[2] >> 64) as u64); - let (z2_2l, z2_2h) = mul64( - b2 as u64 ^ (b2 >> 64) as u64, - h_powers_rev[2] as u64 ^ (h_powers_rev[2] >> 64) as u64, - ); + let (z0_2l, z0_2h) = mul64(b2_lo, h2_lo); + let (z1_2l, z1_2h) = mul64(b2_hi, h2_hi); + let (z2_2l, z2_2h) = mul64(b2_lo ^ b2_hi, h2_lo ^ h2_hi); - let (z0_3l, z0_3h) = mul64(b3 as u64, h_powers_rev[3] as u64); - let (z1_3l, z1_3h) = mul64((b3 >> 64) as u64, (h_powers_rev[3] >> 64) as u64); - let (z2_3l, z2_3h) = mul64( - b3 as u64 ^ (b3 >> 64) as u64, - h_powers_rev[3] as u64 ^ (h_powers_rev[3] >> 64) as u64, - ); + let (z0_3l, z0_3h) = mul64(b3_lo, h3_lo); + let (z1_3l, z1_3h) = mul64(b3_hi, h3_hi); + let (z2_3l, z2_3h) = mul64(b3_lo ^ b3_hi, h3_lo ^ h3_hi); // XOR all 4 Karatsuba intermediates. let z0_lo = z0_0l ^ z0_1l ^ z0_2l ^ z0_3l; @@ -794,7 +859,6 @@ mod ppc_vpmsum { // riscv64 Zvbc backend (vector carryless multiply) #[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] mod rv_clmul { use core::arch::asm; @@ -802,6 +866,9 @@ mod rv_clmul { /// /// Uses the clobber-only vreg workaround: data shuttled through GPRs /// and memory; vector registers referenced by explicit names. + /// + /// # Safety + /// Caller must ensure the V and Zvbc extensions are available. #[inline] #[target_feature(enable = "v", enable = "zvbc")] unsafe fn mul64(a: u64, b: u64) -> (u64, u64) { @@ -840,10 +907,8 @@ mod rv_clmul { pub(super) unsafe fn clmul128_reduce(a: u128, b: u128) -> u128 { // SAFETY: target_feature gate guarantees Zvbc availability. unsafe { - let a_lo = a as u64; - let a_hi = (a >> 64) as u64; - let b_lo = b as u64; - let b_hi = (b >> 64) as u64; + let (a_lo, a_hi) = super::split_u128_le(a); + let (b_lo, b_hi) = super::split_u128_le(b); // Karatsuba: 3 multiplies. let (v0_lo, v0_hi) = mul64(a_lo, b_lo); @@ -869,13 +934,15 @@ mod rv_clmul { // riscv64 Zbc backend (scalar carryless multiply) #[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] mod rv_scalar_clmul { use core::arch::asm; /// 64×64→128 carryless multiply using scalar Zbc (clmul + clmulh). /// /// Identical encoding to Zbkc — dispatch checks either cap at runtime. + /// + /// # Safety + /// Caller must ensure the Zbc or Zbkc extension is available. #[inline] #[target_feature(enable = "zbc")] unsafe fn mul64(a: u64, b: u64) -> (u64, u64) { @@ -909,10 +976,8 @@ mod rv_scalar_clmul { // SAFETY: target_feature gate guarantees Zbc availability. mul64 calls // are safe within this target_feature scope. unsafe { - let a_lo = a as u64; - let a_hi = (a >> 64) as u64; - let b_lo = b as u64; - let b_hi = (b >> 64) as u64; + let (a_lo, a_hi) = super::split_u128_le(a); + let (b_lo, b_hi) = super::split_u128_le(b); // Karatsuba: 3 multiplies instead of 4. let (v0_lo, v0_hi) = mul64(a_lo, b_lo); @@ -981,7 +1046,7 @@ mod vpclmul { let result = super::pclmul::mont_reduce_sse2(lo_sum, hi_sum); let mut out = 0u128; - _mm_storeu_si128((&mut out as *mut u128).cast(), result); + _mm_storeu_si128(core::ptr::from_mut(&mut out).cast(), result); out } } @@ -1039,57 +1104,7 @@ mod vpclmul { let result = super::pclmul::mont_reduce_sse2(lo, hi); let mut out = 0u128; - _mm_storeu_si128((&mut out as *mut u128).cast(), result); - out - } - } - - /// Reduce eight POLYVAL-domain lanes using 256-bit VPCLMULQDQ. - /// - /// # Safety - /// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VPCLMULQDQ + - /// PCLMULQDQ + SSE2. - #[cfg(feature = "aes-gcm")] - #[target_feature(enable = "avx2,avx512f,avx512vl,vpclmulqdq,pclmulqdq,sse2")] - #[inline] - #[allow(dead_code)] - unsafe fn aggregate_8_lanes_256(data: [__m256i; 4], h_powers_rev: &[u128; 8]) -> u128 { - // SAFETY: x86 VPCLMUL 8-block aggregation because: - // 1. This function's caller guarantees all required target features. - // 2. `data` contains eight initialized POLYVAL-domain lanes in order. - // 3. `h_powers_rev` contains exactly [H^8, H^7, ..., H], matching the lanes. - unsafe { - let mut lo_sum = _mm256_setzero_si256(); - let mut hi_sum = _mm256_setzero_si256(); - - macro_rules! fold_lanes { - ($data:expr, $power_offset:expr) => {{ - let h_vec = _mm256_loadu_si256(h_powers_rev.as_ptr().add($power_offset).cast()); - let lo = _mm256_clmulepi64_epi128($data, h_vec, 0x00); - let hi = _mm256_clmulepi64_epi128($data, h_vec, 0x11); - let data_mid = _mm256_xor_si256($data, _mm256_shuffle_epi32::<0x4e>($data)); - let h_mid = _mm256_xor_si256(h_vec, _mm256_shuffle_epi32::<0x4e>(h_vec)); - let mid = _mm256_xor_si256( - _mm256_xor_si256(_mm256_clmulepi64_epi128(data_mid, h_mid, 0x00), lo), - hi, - ); - - lo_sum = _mm256_xor_si256(lo_sum, _mm256_xor_si256(lo, _mm256_bslli_epi128(mid, 8))); - hi_sum = _mm256_xor_si256(hi_sum, _mm256_xor_si256(hi, _mm256_bsrli_epi128(mid, 8))); - }}; - } - - fold_lanes!(data[0], 0); - fold_lanes!(data[1], 2); - fold_lanes!(data[2], 4); - fold_lanes!(data[3], 6); - - let lo = _mm_xor_si128(_mm256_castsi256_si128(lo_sum), _mm256_extracti128_si256(lo_sum, 1)); - let hi = _mm_xor_si128(_mm256_castsi256_si128(hi_sum), _mm256_extracti128_si256(hi_sum, 1)); - - let result = super::pclmul::mont_reduce_sse2(lo, hi); - let mut out = 0u128; - _mm_storeu_si128((&mut out as *mut u128).cast(), result); + _mm_storeu_si128(core::ptr::from_mut(&mut out).cast(), result); out } } @@ -1123,28 +1138,6 @@ mod vpclmul { _mm512_shuffle_epi8(raw, reverse_bytes) } - /// Convert two big-endian GHASH lanes into POLYVAL-domain lanes. - /// - /// # Safety - /// Caller must ensure AVX2 is available. - #[cfg(feature = "aes-gcm")] - #[target_feature(enable = "avx2")] - #[inline] - #[allow(dead_code)] - unsafe fn be_lanes_256(raw: __m256i) -> __m256i { - let reverse_bytes = _mm256_set_epi32( - 0x0001_0203, - 0x0405_0607, - 0x0809_0a0b, - 0x0c0d_0e0f, - 0x0001_0203, - 0x0405_0607, - 0x0809_0a0b, - 0x0c0d_0e0f, - ); - _mm256_shuffle_epi8(raw, reverse_bytes) - } - /// Convert four big-endian GHASH lanes and XOR `acc` into the first lane. /// /// # Safety @@ -1158,30 +1151,11 @@ mod vpclmul { // 2. `raw` contains four initialized big-endian GHASH lanes and `acc` is initialized. unsafe { let data = be_lanes(raw); - let acc_lane = _mm512_zextsi128_si512(_mm_loadu_si128((&acc as *const u128).cast())); + let acc_lane = _mm512_zextsi128_si512(_mm_loadu_si128(core::ptr::from_ref(&acc).cast())); _mm512_xor_si512(data, acc_lane) } } - /// Convert two big-endian GHASH lanes and XOR `acc` into the first lane. - /// - /// # Safety - /// Caller must ensure AVX2 and SSE2 are available. - #[cfg(feature = "aes-gcm")] - #[target_feature(enable = "avx2,sse2")] - #[inline] - #[allow(dead_code)] - unsafe fn be_lanes_256_with_acc(acc: u128, raw: __m256i) -> __m256i { - // SAFETY: direct GHASH lane conversion because: - // 1. This function's caller guarantees the required x86 target features. - // 2. `raw` contains two initialized big-endian GHASH lanes and `acc` is initialized. - unsafe { - let data = be_lanes_256(raw); - let acc_lane = _mm256_castsi128_si256(_mm_loadu_si128((&acc as *const u128).cast())); - _mm256_xor_si256(data, acc_lane) - } - } - /// Load four big-endian GHASH blocks and XOR `acc` into the first lane. /// /// # Safety @@ -1272,7 +1246,7 @@ mod vpclmul { // representation expected by `aggregate_lanes`. unsafe { let data = _mm512_loadu_si512(block_ptr.cast()); - let acc_lane = _mm512_zextsi128_si512(_mm_loadu_si128((&acc as *const u128).cast())); + let acc_lane = _mm512_zextsi128_si512(_mm_loadu_si128(core::ptr::from_ref(&acc).cast())); aggregate_lanes(_mm512_xor_si512(data, acc_lane), h_powers_rev) } } @@ -1292,7 +1266,7 @@ mod vpclmul { // 3. Only the first lane receives the incoming accumulator, matching the POLYVAL recurrence. unsafe { let data0 = _mm512_loadu_si512(block_ptr.cast()); - let acc_lane = _mm512_zextsi128_si512(_mm_loadu_si128((&acc as *const u128).cast())); + let acc_lane = _mm512_zextsi128_si512(_mm_loadu_si128(core::ptr::from_ref(&acc).cast())); aggregate_16_lanes( [ _mm512_xor_si512(data0, acc_lane), @@ -1368,39 +1342,6 @@ mod vpclmul { ) } } - - /// Process 8 big-endian GHASH lanes already resident in 256-bit SIMD registers. - /// - /// # Safety - /// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VPCLMULQDQ + - /// PCLMULQDQ + SSE2. - #[cfg(feature = "aes-gcm")] - #[target_feature(enable = "avx2,avx512f,avx512vl,vpclmulqdq,pclmulqdq,sse2")] - #[allow(dead_code)] - pub(super) unsafe fn aggregate_8blocks_be_lanes_256( - acc: u128, - h_powers_rev: &[u128; 8], - raw0: __m256i, - raw1: __m256i, - raw2: __m256i, - raw3: __m256i, - ) -> u128 { - // SAFETY: direct-lane GHASH aggregation because: - // 1. This function's caller guarantees all required x86 target features. - // 2. `raw*` contain eight initialized 16-byte ciphertext lanes in memory byte order. - // 3. Only the first lane receives the incoming accumulator, matching GHASH recurrence. - unsafe { - aggregate_8_lanes_256( - [ - be_lanes_256_with_acc(acc, raw0), - be_lanes_256(raw1), - be_lanes_256(raw2), - be_lanes_256(raw3), - ], - h_powers_rev, - ) - } - } } // x86_64: inline helpers for fused GCM paths @@ -1561,29 +1502,6 @@ pub(super) unsafe fn x86_aggregate_16blocks_be_lanes_inline( unsafe { vpclmul::aggregate_16blocks_be_lanes(acc, h_powers_rev, raw0, raw1, raw2, raw3) } } -/// 8-block VPCLMULQDQ aggregate helper for big-endian GHASH lanes already in 256-bit registers. -/// -/// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VPCLMULQDQ + -/// PCLMULQDQ + SSE2 are available. -#[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] -#[target_feature(enable = "avx2,avx512f,avx512vl,vpclmulqdq,pclmulqdq,sse2")] -#[inline] -pub(super) unsafe fn x86_aggregate_8blocks_be_lanes_256_inline( - acc: u128, - h_powers_rev: &[u128; 8], - raw0: core::arch::x86_64::__m256i, - raw1: core::arch::x86_64::__m256i, - raw2: core::arch::x86_64::__m256i, - raw3: core::arch::x86_64::__m256i, -) -> u128 { - // SAFETY: direct-lane VPCLMUL aggregation because: - // 1. Caller guarantees all required target features. - // 2. `h_powers_rev` and `raw*` are initialized inputs matching the backend contract. - // 3. The helper folds the incoming accumulator only into the first lane. - unsafe { vpclmul::aggregate_8blocks_be_lanes_256(acc, h_powers_rev, raw0, raw1, raw2, raw3) } -} - // aarch64: inline helper for fused paths (#[target_feature] + #[inline(always)]) /// PMULL-based 128×128 carryless multiply + Montgomery reduce, guaranteed @@ -1848,6 +1766,8 @@ pub(super) fn clmul128_reduce(a: u128, b: u128) -> u128 { clmul(a, b) } +/// Multiplies two little-endian POLYVAL field elements with the portable +/// carryless-multiply and reduction implementation. #[cfg(all(feature = "diag", feature = "aes-gcm-siv"))] #[must_use] pub fn diag_polyval_reduce_portable(a: &[u8; 16], b: &[u8; 16]) -> [u8; 16] { @@ -1968,7 +1888,8 @@ pub(super) fn precompute_powers_128(h: u128) -> [u128; 128] { pub(super) fn precompute_powers_16_mid(h_powers_rev_16: &[u128; 16]) -> [u128; 16] { core::array::from_fn(|i| { let h = h_powers_rev_16[i]; - ((h as u64) ^ ((h >> 64) as u64)) as u128 + let (lo, hi) = split_u128_le(h); + (lo ^ hi) as u128 }) } @@ -1985,10 +1906,8 @@ pub(super) fn precompute_powers_16_pair(h_powers_rev_16: &[u128; 16]) -> [u128; let lane = i % 3; let h0 = h_powers_rev_16[pair.strict_mul(2)]; let h1 = h_powers_rev_16[pair.strict_mul(2).strict_add(1)]; - let lo0 = h0 as u64; - let hi0 = (h0 >> 64) as u64; - let lo1 = h1 as u64; - let hi1 = (h1 >> 64) as u64; + let (lo0, hi0) = split_u128_le(h0); + let (lo1, hi1) = split_u128_le(h1); let low = match lane { 0 => lo0, @@ -2251,10 +2170,11 @@ impl Polyval { #[cfg(feature = "aes-gcm-siv")] impl Drop for Polyval { fn drop(&mut self) { - // SAFETY: self.acc/self.h are valid, aligned, dereferenceable pointers to initialized memory. + // SAFETY: the raw pointers address initialized, aligned `u128` fields owned + // exclusively by `self` for the duration of `drop`. unsafe { - core::ptr::write_volatile(&mut self.acc, 0); - core::ptr::write_volatile(&mut self.h, 0); + core::ptr::write_volatile(&raw mut self.acc, 0); + core::ptr::write_volatile(&raw mut self.h, 0); } core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } @@ -2313,10 +2233,8 @@ fn bmul64(x: u64, y: u64) -> u64 { /// Uses Karatsuba decomposition: 3 sub-products × 2 (normal + bit-reversed /// for the high halves) = 6 bmul64 calls = 96 integer multiplies. pub(super) fn clmul128(a: u128, b: u128) -> [u64; 4] { - let a0 = a as u64; - let a1 = (a >> 64) as u64; - let b0 = b as u64; - let b1 = (b >> 64) as u64; + let (a0, a1) = split_u128_le(a); + let (b0, b1) = split_u128_le(b); // Karatsuba middle term operands. let a2 = a0 ^ a1; @@ -2395,21 +2313,24 @@ pub(super) fn mont_reduce(v: [u64; 4]) -> u128 { mod tests { use super::*; + const RFC_H: [u8; 16] = 0x2562_9347_5892_4276_1d31_f826_ba4b_757bu128.to_be_bytes(); + const RFC_X1: [u8; 16] = 0x4f4f_9566_8c83_dfb6_4017_62bb_2d01_a262u128.to_be_bytes(); + const RFC_X2: [u8; 16] = 0xd1a2_4ddd_2721_d006_bbe4_5f20_d3c9_f362u128.to_be_bytes(); + #[cfg(feature = "aes-gcm-siv")] + const RFC_EXPECTED: [u8; 16] = 0xf7a3_b47b_8461_19fa_e5b7_866c_f5e5_b77eu128.to_be_bytes(); + const AGGREGATE_X3: [u8; 16] = 0x0100_0000_0000_0000_0000_0000_0000_000fu128.to_be_bytes(); + const AGGREGATE_X4: [u8; 16] = 0xabcd_ef01_2345_6789_abcd_ef01_2345_6789u128.to_be_bytes(); + /// RFC 8452 Appendix A: POLYVAL test vector. #[cfg(feature = "aes-gcm-siv")] #[test] fn polyval_rfc8452_appendix_a() { - let h = hex_to_16("25629347589242761d31f826ba4b757b"); - let x1 = hex_to_16("4f4f95668c83dfb6401762bb2d01a262"); - let x2 = hex_to_16("d1a24ddd2721d006bbe45f20d3c9f362"); - let expected = hex_to_16("f7a3b47b846119fae5b7866cf5e5b77e"); - - let mut pv = Polyval::new(&h); - pv.update_block(&x1); - pv.update_block(&x2); + let mut pv = Polyval::new(&RFC_H); + pv.update_block(&RFC_X1); + pv.update_block(&RFC_X2); let result = pv.finalize(); - assert_eq!(result, expected, "POLYVAL mismatch"); + assert_eq!(result, RFC_EXPECTED, "POLYVAL mismatch"); } /// POLYVAL with empty input should return zero. @@ -2436,26 +2357,26 @@ mod tests { #[cfg(feature = "aes-gcm-siv")] #[test] fn polyval_padded_matches_manual() { - let h = hex_to_16("25629347589242761d31f826ba4b757b"); let data = b"Hello, World! This is test data for POLYVAL padding."; // Manual: split into 16-byte blocks, pad last one. - let mut manual = Polyval::new(&h); - let mut offset = 0; - while offset + 16 <= data.len() { - let block: [u8; 16] = data[offset..offset + 16].try_into().unwrap(); + let mut manual = Polyval::new(&RFC_H); + let mut chunks = data.chunks_exact(BLOCK_SIZE); + for chunk in chunks.by_ref() { + let mut block = [0u8; BLOCK_SIZE]; + block.copy_from_slice(chunk); manual.update_block(&block); - offset += 16; } - if offset < data.len() { - let mut block = [0u8; 16]; - block[..data.len() - offset].copy_from_slice(&data[offset..]); + let remainder = chunks.remainder(); + if !remainder.is_empty() { + let mut block = [0u8; BLOCK_SIZE]; + block[..remainder.len()].copy_from_slice(remainder); manual.update_block(&block); } let manual_result = manual.finalize(); // Padded API. - let mut padded = Polyval::new(&h); + let mut padded = Polyval::new(&RFC_H); padded.update_padded(data); let padded_result = padded.finalize(); @@ -2498,8 +2419,9 @@ mod tests { fn clmul128_by_one() { let val: u128 = 0x7b75_4bba_26f8_311d_7642_9258_4793_6225; let v = clmul128(1, val); - assert_eq!(v[0], val as u64, "v0 should be val_lo"); - assert_eq!(v[1], (val >> 64) as u64, "v1 should be val_hi"); + let (val_lo, val_hi) = split_u128_le(val); + assert_eq!(v[0], val_lo, "v0 should be val_lo"); + assert_eq!(v[1], val_hi, "v1 should be val_hi"); assert_eq!(v[2], 0, "v2 should be 0"); assert_eq!(v[3], 0, "v3 should be 0"); } @@ -2508,7 +2430,8 @@ mod tests { /// in the low half gives x^128 * x^{-128} = 1. #[test] fn mont_reduce_of_poly() { - let v = [POLY as u64, (POLY >> 64) as u64, 0u64, 0u64]; + let (poly_lo, poly_hi) = split_u128_le(POLY); + let v = [poly_lo, poly_hi, 0u64, 0u64]; let result = mont_reduce(v); assert_eq!(result, 1, "mont_reduce(POLY) should be 1"); } @@ -2516,7 +2439,7 @@ mod tests { /// Verify precompute_powers produces correct powers of H. #[test] fn precompute_powers_correct() { - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers(h); assert_eq!(powers[0], h, "powers[0] should be H"); assert_eq!(powers[1], clmul128_reduce(h, h), "powers[1] should be H^2"); @@ -2527,7 +2450,7 @@ mod tests { /// Verify precompute_powers_16 extends the same power chain to H^16. #[test] fn precompute_powers_16_correct() { - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers_16(h); assert_eq!(powers[0], h, "powers[0] should be H"); @@ -2545,7 +2468,7 @@ mod tests { /// Verify precompute_powers_32 extends the same power chain to H^32. #[test] fn precompute_powers_32_correct() { - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers_32(h); assert_eq!(powers[0], h, "powers[0] should be H"); @@ -2563,7 +2486,7 @@ mod tests { /// Verify precompute_powers_64 extends the same power chain to H^64. #[test] fn precompute_powers_64_correct() { - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers_64(h); assert_eq!(powers[0], h, "powers[0] should be H"); @@ -2581,7 +2504,7 @@ mod tests { /// Verify precompute_powers_128 extends the same power chain to H^128. #[test] fn precompute_powers_128_correct() { - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers_128(h); assert_eq!(powers[0], h, "powers[0] should be H"); @@ -2599,16 +2522,15 @@ mod tests { /// Verify accumulate_4blocks matches sequential block-by-block processing. #[test] fn accumulate_4blocks_matches_sequential() { - let h_bytes = hex_to_16("25629347589242761d31f826ba4b757b"); - let h = u128::from_le_bytes(h_bytes); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers(h); let h_powers_rev = [powers[3], powers[2], powers[1], powers[0]]; let blocks = [ - u128::from_le_bytes(hex_to_16("4f4f95668c83dfb6401762bb2d01a262")), - u128::from_le_bytes(hex_to_16("d1a24ddd2721d006bbe45f20d3c9f362")), - u128::from_le_bytes(hex_to_16("0100000000000000000000000000000f")), - u128::from_le_bytes(hex_to_16("abcdef0123456789abcdef0123456789")), + u128::from_le_bytes(RFC_X1), + u128::from_le_bytes(RFC_X2), + u128::from_le_bytes(AGGREGATE_X3), + u128::from_le_bytes(AGGREGATE_X4), ]; let acc = 0x42u128; @@ -2631,15 +2553,18 @@ mod tests { /// Verify accumulate_16blocks matches sequential block-by-block processing. #[test] fn accumulate_16blocks_matches_sequential() { - let h_bytes = hex_to_16("25629347589242761d31f826ba4b757b"); - let h = u128::from_le_bytes(h_bytes); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers_16(h); let h_powers_rev = core::array::from_fn(|i| powers[15usize.strict_sub(i)]); - let blocks = core::array::from_fn(|i| { - let lane = (i as u128).wrapping_add(1); - 0x4f4f_9566_8c83_dfb6_4017_62bb_2d01_a262u128.wrapping_mul(lane) - ^ 0xd1a2_4ddd_2721_d006_bbe4_5f20_d3c9_f362u128.rotate_left(i as u32) - }); + let mut blocks = [0u128; 16]; + let mut lane = 1u128; + let mut rotation = 0u32; + for block in &mut blocks { + *block = 0x4f4f_9566_8c83_dfb6_4017_62bb_2d01_a262u128.wrapping_mul(lane) + ^ 0xd1a2_4ddd_2721_d006_bbe4_5f20_d3c9_f362u128.rotate_left(rotation); + lane = lane.wrapping_add(1); + rotation = rotation.strict_add(1); + } let acc = 0x42u128; let mut seq = acc; @@ -2654,76 +2579,11 @@ mod tests { assert_eq!(wide, seq, "16-block aggregate must match sequential processing"); } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[target_feature(enable = "avx2,avx512f,avx512vl,vpclmulqdq,pclmulqdq,sse2")] - /// # Safety - /// - /// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VPCLMULQDQ + - /// PCLMULQDQ + SSE2 are available. - unsafe fn x86_aggregate_8blocks_be_lanes_256_test_call( - acc: u128, - h_powers_rev: &[u128; 8], - bytes: &[u8; 128], - ) -> u128 { - use core::arch::x86_64::*; - - // SAFETY: test-only x86 lane aggregation because: - // 1. The caller verified the CPU features required by this target-feature helper. - // 2. `bytes` is exactly 128 initialized bytes, so all four 32-byte loads are in bounds. - // 3. The loaded lanes are passed directly to the helper under test. - unsafe { - let raw0 = _mm256_loadu_si256(bytes.as_ptr().cast()); - let raw1 = _mm256_loadu_si256(bytes.as_ptr().add(32).cast()); - let raw2 = _mm256_loadu_si256(bytes.as_ptr().add(64).cast()); - let raw3 = _mm256_loadu_si256(bytes.as_ptr().add(96).cast()); - x86_aggregate_8blocks_be_lanes_256_inline(acc, h_powers_rev, raw0, raw1, raw2, raw3) - } - } - - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[test] - fn x86_aggregate_8blocks_be_lanes_256_matches_sequential() { - let required = crate::platform::caps::x86::VPCLMUL_READY | crate::platform::caps::x86::AVX2; - if !crate::platform::caps().has(required) { - return; - } - - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); - let powers = precompute_powers_8(h); - let h_powers_rev = core::array::from_fn(|i| powers[7usize.strict_sub(i)]); - let acc = 0x1122_3344_5566_7788_99aa_bbcc_ddee_ff00u128; - - let mut bytes = [0u8; 128]; - let mut i = 0usize; - while i < bytes.len() { - bytes[i] = i.wrapping_mul(37).wrapping_add(19) as u8; - i = i.strict_add(1); - } - - let mut expected = acc; - let mut offset = 0usize; - while offset < bytes.len() { - let mut block = [0u8; 16]; - block.copy_from_slice(&bytes[offset..offset.strict_add(16)]); - expected ^= u128::from_be_bytes(block); - expected = clmul128_reduce(expected, h); - offset = offset.strict_add(16); - } - - // SAFETY: Runtime caps above confirmed AVX2 + VPCLMUL_READY before calling the target-feature - // helper. The byte array and H-power table are fully initialized. - let wide = unsafe { x86_aggregate_8blocks_be_lanes_256_test_call(acc, &h_powers_rev, &bytes) }; - assert_eq!( - wide, expected, - "8-block 256-bit VPCLMUL GHASH aggregate must match sequential fold" - ); - } - /// Verify the x86 padded wide path matches scalar POLYVAL over boundary sizes. #[cfg(all(target_arch = "x86_64", feature = "aes-gcm-siv"))] #[test] fn accumulate_padded_x86_matches_sequential_boundaries() { - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers_16(h); let h_powers_rev = [powers[3], powers[2], powers[1], powers[0]]; let h_powers_rev_16 = core::array::from_fn(|i| powers[15usize.strict_sub(i)]); @@ -2771,14 +2631,4 @@ mod tests { let result = accumulate_4blocks(0, h, &h_powers_rev, &blocks); assert_eq!(result, 0); } - - fn hex_to_16(hex: &str) -> [u8; 16] { - let mut out = [0u8; 16]; - let mut i = 0; - while i < 16 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - i = i.strict_add(1); - } - out - } } diff --git a/src/aead/targets.rs b/src/aead/targets.rs index 709e29d9..cf3e2c85 100644 --- a/src/aead/targets.rs +++ b/src/aead/targets.rs @@ -5,86 +5,240 @@ //! //! Backend selection is derived from detected CPU capabilities. +#[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] +use crate::platform::caps::wasm; use crate::platform::{ Arch, Caps, - caps::{aarch64, power, riscv, s390x, wasm, x86}, + caps::{aarch64, power, riscv, s390x, x86}, }; /// AEAD primitives on the public surface. -#[allow(dead_code)] // Reduced-feature test builds can compile only byte wrappers, not live dispatch. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[non_exhaustive] -pub enum AeadPrimitive { +pub(super) enum AeadPrimitive { + #[cfg(any(test, feature = "xchacha20poly1305"))] XChaCha20Poly1305, + #[cfg(any(test, feature = "chacha20poly1305"))] ChaCha20Poly1305, + #[cfg(any( + test, + all( + feature = "aes-gcm-siv", + any( + feature = "diag", + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Aes256GcmSiv, + #[cfg(any(test, feature = "aes-gcm"))] Aes256Gcm, + #[cfg(any(test, feature = "aes-gcm"))] Aes128Gcm, + #[cfg(any( + test, + all( + feature = "aes-gcm-siv", + any( + feature = "diag", + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Aes128GcmSiv, - AsconAead128, + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Aegis256, } /// Backend classes selected by live dispatch. -#[allow(dead_code)] // Some architecture-specific variants are only constructed on their target. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[non_exhaustive] -pub enum AeadBackend { +pub(super) enum AeadBackend { Portable, WasmPortable, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] WasmSimd128, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] X86Avx2, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] X86Avx512, + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] X86Aesni, + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] X86AesniPclmul, + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] X86VaesVpclmul, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Aarch64Neon, + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Aarch64Aes, + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Aarch64AesPmull, + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Aarch64Sve2AesPmull, + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] S390xMsa, /// Hamburg vperm AES rounds for AEGIS using register-only z/Vector VPERM. /// Used on s390x z13+ where no single-round AES instruction exists. + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] S390xVperm, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] S390xVector, + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] Power8Crypto, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] PowerVector, + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] Riscv64ScalarCrypto, + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] Riscv64VectorCrypto, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Riscv64Vector, - /// Hamburg vperm AES via register-only `vrgather.vv` operations. - /// Kept as an explicit backend, but not selected for V-only RISC-V without - /// target-native evidence supporting that dispatch policy. - Riscv64Vperm, } impl AeadBackend { /// Stable backend label for diagnostics and future benchmark grouping. - #[allow(dead_code)] // Used by `diag`; reduced-feature builds can compile dispatch without introspection. + #[cfg(feature = "diag")] #[must_use] - pub const fn name(self) -> &'static str { + pub(super) const fn name(self) -> &'static str { match self { Self::Portable => "portable", Self::WasmPortable => "wasm32/portable", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::WasmSimd128 => "wasm32/simd128", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::X86Avx2 => "x86_64/avx2", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::X86Avx512 => "x86_64/avx512", + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Self::X86Aesni => "x86_64/aesni", + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::X86AesniPclmul => "x86_64/aesni+pclmul", + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::X86VaesVpclmul => "x86_64/vaes+vpclmul", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::Aarch64Neon => "aarch64/neon", + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Self::Aarch64Aes => "aarch64/aes", + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::Aarch64AesPmull => "aarch64/aes+pmull", + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::Aarch64Sve2AesPmull => "aarch64/sve2+aes+pmull", + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::S390xMsa => "s390x/msa", + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Self::S390xVperm => "s390x/vperm", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::S390xVector => "s390x/vector", + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::Power8Crypto => "powerpc64/crypto", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::PowerVector => "powerpc64/vector", + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::Riscv64ScalarCrypto => "riscv64/scalar-crypto", + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::Riscv64VectorCrypto => "riscv64/vector-crypto", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::Riscv64Vector => "riscv64/vector", - Self::Riscv64Vperm => "riscv64/vperm", } } } @@ -93,21 +247,50 @@ impl AeadBackend { /// and current dispatch policy. /// /// SIMD classes without accepted target-native evidence resolve to `portable`. -#[allow(dead_code)] // Some leaf AEAD builds compile target policy without live dispatch on the host. #[must_use] -pub fn select_backend(primitive: AeadPrimitive, arch: Arch, caps: Caps) -> AeadBackend { +pub(super) fn select_backend(primitive: AeadPrimitive, arch: Arch, caps: Caps) -> AeadBackend { match primitive { - AeadPrimitive::XChaCha20Poly1305 | AeadPrimitive::ChaCha20Poly1305 => select_chacha_backend(arch, caps), - AeadPrimitive::Aes256GcmSiv | AeadPrimitive::Aes256Gcm | AeadPrimitive::Aes128Gcm | AeadPrimitive::Aes128GcmSiv => { - select_gcm_backend(arch, caps) - } - AeadPrimitive::AsconAead128 => select_ascon_backend(arch), + #[cfg(any(test, feature = "xchacha20poly1305"))] + AeadPrimitive::XChaCha20Poly1305 => select_chacha_backend(arch, caps), + #[cfg(any(test, feature = "chacha20poly1305"))] + AeadPrimitive::ChaCha20Poly1305 => select_chacha_backend(arch, caps), + #[cfg(any( + test, + all( + feature = "aes-gcm-siv", + any( + feature = "diag", + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] + AeadPrimitive::Aes256GcmSiv | AeadPrimitive::Aes128GcmSiv => select_gcm_backend(arch, caps), + #[cfg(any(test, feature = "aes-gcm"))] + AeadPrimitive::Aes256Gcm | AeadPrimitive::Aes128Gcm => select_gcm_backend(arch, caps), + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] AeadPrimitive::Aegis256 => select_aegis_backend(arch, caps), } } +#[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] #[inline] -#[allow(dead_code)] // Only used when a ChaCha20-Poly1305 feature is enabled. fn select_chacha_backend(arch: Arch, caps: Caps) -> AeadBackend { match arch { Arch::X86_64 => { @@ -158,8 +341,8 @@ fn select_chacha_backend(arch: Arch, caps: Caps) -> AeadBackend { } } +#[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] #[inline] -#[allow(dead_code)] // Only used when an AES-GCM-family feature needs live dispatch on this target. fn select_gcm_backend(arch: Arch, caps: Caps) -> AeadBackend { match arch { Arch::X86_64 => { @@ -214,24 +397,28 @@ fn select_gcm_backend(arch: Arch, caps: Caps) -> AeadBackend { } } +#[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) +))] #[inline] -#[allow(dead_code)] // Only used by Ascon-AEAD dispatch diagnostics. -const fn select_ascon_backend(arch: Arch) -> AeadBackend { - match arch { - Arch::Wasm32 | Arch::Wasm64 => AeadBackend::WasmPortable, - _ => AeadBackend::Portable, - } -} - -#[inline] -#[allow(dead_code)] // Only used when AEGIS-256 needs target-policy dispatch on this target. fn select_aegis_backend(arch: Arch, caps: Caps) -> AeadBackend { // VAES-256 is intentionally not used for AEGIS-256. Its six dependent state // lanes require cross-lane shuffles in the packed representation; see the // XMM-state path in aegis256.rs. match arch { Arch::X86_64 => { - if caps.has(x86::AESNI) { + if caps.has(x86::AESNI) && caps.has(x86::AVX) { AeadBackend::X86Aesni } else { AeadBackend::Portable @@ -382,6 +569,10 @@ mod tests { ); assert_eq!( select_backend(AeadPrimitive::Aegis256, Arch::X86_64, x86::AESNI), + AeadBackend::Portable + ); + assert_eq!( + select_backend(AeadPrimitive::Aegis256, Arch::X86_64, x86::AESNI | x86::AVX), AeadBackend::X86Aesni ); assert_eq!( @@ -562,16 +753,4 @@ mod tests { AeadBackend::WasmPortable ); } - - #[test] - fn ascon_stays_portable_until_simd_policy_is_accepted() { - assert_eq!( - select_backend(AeadPrimitive::AsconAead128, Arch::X86_64, x86::AVX2 | x86::VAES_READY), - AeadBackend::Portable - ); - assert_eq!( - select_backend(AeadPrimitive::AsconAead128, Arch::Wasm32, Caps::NONE), - AeadBackend::WasmPortable - ); - } } diff --git a/src/aead/xchacha20poly1305.rs b/src/aead/xchacha20poly1305.rs index 8e1cd870..8922a552 100644 --- a/src/aead/xchacha20poly1305.rs +++ b/src/aead/xchacha20poly1305.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! XChaCha20-Poly1305 public AEAD surface. use core::fmt; @@ -220,8 +218,12 @@ mod tests { let cipher = XChaCha20Poly1305::new(&key); let mut buf = *b"hello xchacha"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); - cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &tag).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("XChaCha20-Poly1305 encryption must succeed"); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buf, &tag) + .expect("XChaCha20-Poly1305 decryption must succeed"); assert_eq!(&buf, b"hello xchacha"); } @@ -232,11 +234,15 @@ mod tests { let cipher = XChaCha20Poly1305::new(&key); let mut buf = *b"nonce test"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("XChaCha20-Poly1305 test setup encryption must succeed"); let wrong_nonce = Nonce192::from_bytes([0x08u8; 24]); - let result = cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } #[test] @@ -246,14 +252,18 @@ mod tests { let cipher = XChaCha20Poly1305::new(&key); let mut buf = *b"zero me on failure"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("XChaCha20-Poly1305 test setup encryption must succeed"); let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0xFF; let bad_tag = XChaCha20Poly1305Tag::from_bytes(bad_tag); - let result = cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert!(buf.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } @@ -264,9 +274,13 @@ mod tests { let cipher = XChaCha20Poly1305::new(&key); let mut buf = *b"aad test"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buf) + .expect("XChaCha20-Poly1305 test setup encryption must succeed"); - let result = cipher.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag), + Err(OpenError::verification()) + ); } } diff --git a/src/auth/argon2/aarch64.rs b/src/auth/argon2/aarch64.rs index d0ad0a7a..442c5d15 100644 --- a/src/auth/argon2/aarch64.rs +++ b/src/auth/argon2/aarch64.rs @@ -43,6 +43,17 @@ use core::arch::aarch64::{ use super::BLOCK_WORDS; +struct NeonState { + a_lo: uint64x2_t, + a_hi: uint64x2_t, + b_lo: uint64x2_t, + b_hi: uint64x2_t, + c_lo: uint64x2_t, + c_hi: uint64x2_t, + d_lo: uint64x2_t, + d_hi: uint64x2_t, +} + /// NEON BlaMka compression kernel. /// /// # Safety @@ -63,41 +74,41 @@ pub(super) unsafe fn compress_neon( // value; uint64x2_t has no invalid bit patterns. let mut r = unsafe { [core::mem::zeroed::(); 64] }; let mut q = r; - for i in 0..64 { + for i in 0usize..64 { + let offset = i.strict_mul(2); // SAFETY: x and y are [u64; BLOCK_WORDS] with BLOCK_WORDS == 128; the // loop reads 2 u64 per iteration at offset `2 * i` for i ∈ 0..64, so // the last read is at words 126..=127. NEON inherited from the fn // target_feature. - let (xv, yv) = unsafe { (vld1q_u64(x.as_ptr().add(2 * i)), vld1q_u64(y.as_ptr().add(2 * i))) }; + let (xv, yv) = unsafe { (vld1q_u64(x.as_ptr().add(offset)), vld1q_u64(y.as_ptr().add(offset))) }; r[i] = veorq_u64(xv, yv); q[i] = r[i]; } // Row pass: 8 rows × 16 u64 per row = 8 P-rounds on contiguous blocks // of 8 uint64x2_t. - for row in 0..8 { - let base = row * 8; - let (a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) = ( - q[base], - q[base + 1], - q[base + 2], - q[base + 3], - q[base + 4], - q[base + 5], - q[base + 6], - q[base + 7], - ); + for row in 0usize..8 { + let base = row.strict_mul(8); + let mut state = NeonState { + a_lo: q[base], + a_hi: q[base.strict_add(1)], + b_lo: q[base.strict_add(2)], + b_hi: q[base.strict_add(3)], + c_lo: q[base.strict_add(4)], + c_hi: q[base.strict_add(5)], + d_lo: q[base.strict_add(6)], + d_hi: q[base.strict_add(7)], + }; // SAFETY: NEON inherited from outer target_feature. - let (a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) = - unsafe { p_round_neon(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) }; - q[base] = a_lo; - q[base + 1] = a_hi; - q[base + 2] = b_lo; - q[base + 3] = b_hi; - q[base + 4] = c_lo; - q[base + 5] = c_hi; - q[base + 6] = d_lo; - q[base + 7] = d_hi; + unsafe { p_round_neon(&mut state) }; + q[base] = state.a_lo; + q[base.strict_add(1)] = state.a_hi; + q[base.strict_add(2)] = state.b_lo; + q[base.strict_add(3)] = state.b_hi; + q[base.strict_add(4)] = state.c_lo; + q[base.strict_add(5)] = state.c_hi; + q[base.strict_add(6)] = state.d_lo; + q[base.strict_add(7)] = state.d_hi; } // Column pass: each P-round reads 16 u64s at stride-16 positions @@ -105,41 +116,41 @@ pub(super) unsafe fn compress_neon( // q[col*2+113]`). In uint64x2_t indexing each "col*2" pair corresponds // to a single uint64x2_t at index `col`, and the stride-16 becomes // stride 8 in uint64x2_t space. - for col in 0..8 { - let (a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) = ( - q[col], - q[col + 8], - q[col + 16], - q[col + 24], - q[col + 32], - q[col + 40], - q[col + 48], - q[col + 56], - ); + for col in 0usize..8 { + let mut state = NeonState { + a_lo: q[col], + a_hi: q[col.strict_add(8)], + b_lo: q[col.strict_add(16)], + b_hi: q[col.strict_add(24)], + c_lo: q[col.strict_add(32)], + c_hi: q[col.strict_add(40)], + d_lo: q[col.strict_add(48)], + d_hi: q[col.strict_add(56)], + }; // SAFETY: NEON inherited from outer target_feature. - let (a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) = - unsafe { p_round_neon(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) }; - q[col] = a_lo; - q[col + 8] = a_hi; - q[col + 16] = b_lo; - q[col + 24] = b_hi; - q[col + 32] = c_lo; - q[col + 40] = c_hi; - q[col + 48] = d_lo; - q[col + 56] = d_hi; + unsafe { p_round_neon(&mut state) }; + q[col] = state.a_lo; + q[col.strict_add(8)] = state.a_hi; + q[col.strict_add(16)] = state.b_lo; + q[col.strict_add(24)] = state.b_hi; + q[col.strict_add(32)] = state.c_lo; + q[col.strict_add(40)] = state.c_hi; + q[col.strict_add(48)] = state.d_lo; + q[col.strict_add(56)] = state.d_hi; } // Final XOR with R, fused with the dst store. - for i in 0..64 { + for i in 0usize..64 { let final_v = veorq_u64(q[i], r[i]); // SAFETY: dst[2*i..2*i+2] is within BLOCK_WORDS for i ∈ 0..64. NEON // inherited from the fn target_feature. unsafe { + let offset = i.strict_mul(2); if xor_into { - let cur = vld1q_u64(dst.as_ptr().add(2 * i)); - vst1q_u64(dst.as_mut_ptr().add(2 * i), veorq_u64(cur, final_v)); + let cur = vld1q_u64(dst.as_ptr().add(offset)); + vst1q_u64(dst.as_mut_ptr().add(offset), veorq_u64(cur, final_v)); } else { - vst1q_u64(dst.as_mut_ptr().add(2 * i), final_v); + vst1q_u64(dst.as_mut_ptr().add(offset), final_v); } } } @@ -157,83 +168,48 @@ pub(super) unsafe fn compress_neon( /// aarch64 so any aarch64 build satisfies this, but the `unsafe fn` /// signature preserves the contract for completeness. #[inline(always)] -#[allow(clippy::too_many_arguments)] -unsafe fn p_round_neon( - mut a_lo: uint64x2_t, - mut a_hi: uint64x2_t, - mut b_lo: uint64x2_t, - mut b_hi: uint64x2_t, - mut c_lo: uint64x2_t, - mut c_hi: uint64x2_t, - mut d_lo: uint64x2_t, - mut d_hi: uint64x2_t, -) -> ( - uint64x2_t, - uint64x2_t, - uint64x2_t, - uint64x2_t, - uint64x2_t, - uint64x2_t, - uint64x2_t, - uint64x2_t, -) { +unsafe fn p_round_neon(state: &mut NeonState) { // Column step — 4 parallel GBs. // SAFETY: NEON precondition inherited from caller. - unsafe { - gb_neon( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - } + unsafe { gb_neon(state) }; // Diagonalise: rotate b by 1, c by 2, d by 3 across the 4-lane row. // SAFETY: vextq_u64 only operates on register values, no memory access. unsafe { - let b_lo2 = vextq_u64::<1>(b_lo, b_hi); - let b_hi2 = vextq_u64::<1>(b_hi, b_lo); - b_lo = b_lo2; - b_hi = b_hi2; + let b_lo = vextq_u64::<1>(state.b_lo, state.b_hi); + let b_hi = vextq_u64::<1>(state.b_hi, state.b_lo); + state.b_lo = b_lo; + state.b_hi = b_hi; // c: rotate by 2 == swap lo/hi. - let c_lo2 = c_hi; - let c_hi2 = c_lo; - c_lo = c_lo2; - c_hi = c_hi2; + core::mem::swap(&mut state.c_lo, &mut state.c_hi); - let d_lo2 = vextq_u64::<1>(d_hi, d_lo); - let d_hi2 = vextq_u64::<1>(d_lo, d_hi); - d_lo = d_lo2; - d_hi = d_hi2; + let d_lo = vextq_u64::<1>(state.d_hi, state.d_lo); + let d_hi = vextq_u64::<1>(state.d_lo, state.d_hi); + state.d_lo = d_lo; + state.d_hi = d_hi; } // Diagonal step — 4 parallel GBs on the rotated state. // SAFETY: NEON precondition inherited from caller. - unsafe { - gb_neon( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - } + unsafe { gb_neon(state) }; // Undo diagonalisation: rotate b by -1, c by -2, d by -3. // SAFETY: vextq_u64 only operates on register values. unsafe { - let b_lo2 = vextq_u64::<1>(b_hi, b_lo); - let b_hi2 = vextq_u64::<1>(b_lo, b_hi); - b_lo = b_lo2; - b_hi = b_hi2; + let b_lo = vextq_u64::<1>(state.b_hi, state.b_lo); + let b_hi = vextq_u64::<1>(state.b_lo, state.b_hi); + state.b_lo = b_lo; + state.b_hi = b_hi; // c: undo swap. - let c_lo2 = c_hi; - let c_hi2 = c_lo; - c_lo = c_lo2; - c_hi = c_hi2; + core::mem::swap(&mut state.c_lo, &mut state.c_hi); - let d_lo2 = vextq_u64::<1>(d_lo, d_hi); - let d_hi2 = vextq_u64::<1>(d_hi, d_lo); - d_lo = d_lo2; - d_hi = d_hi2; + let d_lo = vextq_u64::<1>(state.d_lo, state.d_hi); + let d_hi = vextq_u64::<1>(state.d_hi, state.d_lo); + state.d_lo = d_lo; + state.d_hi = d_hi; } - - (a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) } // ─── 4-way parallel BlaMka G ─────────────────────────────────────────────── @@ -246,54 +222,44 @@ unsafe fn p_round_neon( /// Must be called from a context with NEON enabled (inherited from the /// outer `#[target_feature(enable = "neon")]` entry point). #[inline(always)] -#[allow(clippy::too_many_arguments)] -unsafe fn gb_neon( - a_lo: &mut uint64x2_t, - a_hi: &mut uint64x2_t, - b_lo: &mut uint64x2_t, - b_hi: &mut uint64x2_t, - c_lo: &mut uint64x2_t, - c_hi: &mut uint64x2_t, - d_lo: &mut uint64x2_t, - d_hi: &mut uint64x2_t, -) { +unsafe fn gb_neon(state: &mut NeonState) { // SAFETY: NEON precondition inherited; all ops below are register-only. unsafe { // Step 1: a ← a + b + 2·lsb(a)·lsb(b) - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = vaddq_u64(vaddq_u64(*a_lo, *b_lo), p_lo); - *a_hi = vaddq_u64(vaddq_u64(*a_hi, *b_hi), p_hi); + let p_lo = bla_mul(state.a_lo, state.b_lo); + let p_hi = bla_mul(state.a_hi, state.b_hi); + state.a_lo = vaddq_u64(vaddq_u64(state.a_lo, state.b_lo), p_lo); + state.a_hi = vaddq_u64(vaddq_u64(state.a_hi, state.b_hi), p_hi); // d ← (d ^ a) ROR 32 - *d_lo = ror32(veorq_u64(*d_lo, *a_lo)); - *d_hi = ror32(veorq_u64(*d_hi, *a_hi)); + state.d_lo = ror32(veorq_u64(state.d_lo, state.a_lo)); + state.d_hi = ror32(veorq_u64(state.d_hi, state.a_hi)); // Step 2: c ← c + d + 2·lsb(c)·lsb(d) - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = vaddq_u64(vaddq_u64(*c_lo, *d_lo), p_lo); - *c_hi = vaddq_u64(vaddq_u64(*c_hi, *d_hi), p_hi); + let p_lo = bla_mul(state.c_lo, state.d_lo); + let p_hi = bla_mul(state.c_hi, state.d_hi); + state.c_lo = vaddq_u64(vaddq_u64(state.c_lo, state.d_lo), p_lo); + state.c_hi = vaddq_u64(vaddq_u64(state.c_hi, state.d_hi), p_hi); // b ← (b ^ c) ROR 24 - *b_lo = ror24(veorq_u64(*b_lo, *c_lo)); - *b_hi = ror24(veorq_u64(*b_hi, *c_hi)); + state.b_lo = ror24(veorq_u64(state.b_lo, state.c_lo)); + state.b_hi = ror24(veorq_u64(state.b_hi, state.c_hi)); // Step 3: a ← a + b + 2·lsb(a)·lsb(b) - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = vaddq_u64(vaddq_u64(*a_lo, *b_lo), p_lo); - *a_hi = vaddq_u64(vaddq_u64(*a_hi, *b_hi), p_hi); + let p_lo = bla_mul(state.a_lo, state.b_lo); + let p_hi = bla_mul(state.a_hi, state.b_hi); + state.a_lo = vaddq_u64(vaddq_u64(state.a_lo, state.b_lo), p_lo); + state.a_hi = vaddq_u64(vaddq_u64(state.a_hi, state.b_hi), p_hi); // d ← (d ^ a) ROR 16 - *d_lo = ror16(veorq_u64(*d_lo, *a_lo)); - *d_hi = ror16(veorq_u64(*d_hi, *a_hi)); + state.d_lo = ror16(veorq_u64(state.d_lo, state.a_lo)); + state.d_hi = ror16(veorq_u64(state.d_hi, state.a_hi)); // Step 4: c ← c + d + 2·lsb(c)·lsb(d) - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = vaddq_u64(vaddq_u64(*c_lo, *d_lo), p_lo); - *c_hi = vaddq_u64(vaddq_u64(*c_hi, *d_hi), p_hi); + let p_lo = bla_mul(state.c_lo, state.d_lo); + let p_hi = bla_mul(state.c_hi, state.d_hi); + state.c_lo = vaddq_u64(vaddq_u64(state.c_lo, state.d_lo), p_lo); + state.c_hi = vaddq_u64(vaddq_u64(state.c_hi, state.d_hi), p_hi); // b ← (b ^ c) ROR 63 ≡ ROL 1 - *b_lo = ror63(veorq_u64(*b_lo, *c_lo)); - *b_hi = ror63(veorq_u64(*b_hi, *c_hi)); + state.b_lo = ror63(veorq_u64(state.b_lo, state.c_lo)); + state.b_hi = ror63(veorq_u64(state.b_hi, state.c_hi)); } } diff --git a/src/auth/argon2/dispatch.rs b/src/auth/argon2/dispatch.rs index 1df34ae5..8f5cf6bc 100644 --- a/src/auth/argon2/dispatch.rs +++ b/src/auth/argon2/dispatch.rs @@ -221,7 +221,12 @@ mod tests { #[test] fn all_kernels_terminate_on_portable() { - assert_eq!(*ALL_KERNELS.last().unwrap(), KernelId::Portable); + assert_eq!( + *ALL_KERNELS + .last() + .expect("Argon2 kernel table must contain the portable fallback"), + KernelId::Portable + ); } #[test] diff --git a/src/auth/argon2/kernels.rs b/src/auth/argon2/kernels.rs index e2a90811..b4d19a71 100644 --- a/src/auth/argon2/kernels.rs +++ b/src/auth/argon2/kernels.rs @@ -9,7 +9,6 @@ //! in sibling files (`aarch64.rs`, `x86_64.rs`, …) and are gated by //! `#[cfg(target_arch = ...)]`. The runtime dispatcher is [`super::dispatch`]. -#![allow(clippy::indexing_slicing)] // The portable kernel body is generic over the block word count (128) — // every index is bounds-proven by the fixed `[u64; BLOCK_WORDS]` shape. @@ -97,7 +96,7 @@ pub(super) unsafe fn compress_portable( let mut q = r; // Row pass: apply P to each 16-word row (8 rows of 16 u64s). - for chunk in q.chunks_exact_mut(P_LANE_WORDS) { + for chunk in q.as_chunks_mut::().0 { p_direct!( chunk[0], chunk[1], chunk[2], chunk[3], chunk[4], chunk[5], chunk[6], chunk[7], chunk[8], chunk[9], chunk[10], chunk[11], chunk[12], chunk[13], chunk[14], chunk[15], @@ -107,24 +106,24 @@ pub(super) unsafe fn compress_portable( // Column pass: each 16-word column is two u64s per row at the same // register index. for col in 0usize..8 { - let base = col * 2; + let base = col.strict_mul(2); p_direct!( q[base], - q[base + 1], - q[base + 16], - q[base + 17], - q[base + 32], - q[base + 33], - q[base + 48], - q[base + 49], - q[base + 64], - q[base + 65], - q[base + 80], - q[base + 81], - q[base + 96], - q[base + 97], - q[base + 112], - q[base + 113], + q[base.strict_add(1)], + q[base.strict_add(16)], + q[base.strict_add(17)], + q[base.strict_add(32)], + q[base.strict_add(33)], + q[base.strict_add(48)], + q[base.strict_add(49)], + q[base.strict_add(64)], + q[base.strict_add(65)], + q[base.strict_add(80)], + q[base.strict_add(81)], + q[base.strict_add(96)], + q[base.strict_add(97)], + q[base.strict_add(112)], + q[base.strict_add(113)], ); } diff --git a/src/auth/argon2/mod.rs b/src/auth/argon2/mod.rs index d7a79da5..bcc65eea 100644 --- a/src/auth/argon2/mod.rs +++ b/src/auth/argon2/mod.rs @@ -56,8 +56,6 @@ //! //! [owasp-passwords]: https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html -#![allow(clippy::indexing_slicing)] -#![allow(clippy::unwrap_used)] // unwraps here are on slice→array conversions whose lengths are fixed by construction. use alloc::vec::Vec; @@ -141,15 +139,9 @@ impl MemoryBlock { /// repeated byte-slice transmutes. #[inline] fn zeroize_u64_slice_no_fence(words: &mut [u64]) { - let mut chunks = words.chunks_exact_mut(8); - for chunk in &mut chunks { - // SAFETY: chunk has exactly 8 initialized u64 values and [u64; 8] has - // the same alignment requirement as u64. - unsafe { core::ptr::write_volatile(chunk.as_mut_ptr().cast::<[u64; 8]>(), [0u64; 8]) }; - } - for w in chunks.into_remainder() { - // SAFETY: w is a valid, aligned, dereferenceable pointer to initialized u64. - unsafe { core::ptr::write_volatile(w, 0) }; + for word in words { + // SAFETY: `word` is a valid, aligned pointer to an initialized `u64` and remains exclusively borrowed. + unsafe { core::ptr::write_volatile(word, 0) }; } } @@ -171,8 +163,11 @@ fn zeroize_u64_slice(words: &mut [u64]) { /// (RFC 9106 §3.4.1). #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Argon2Variant { + /// Argon2d data-dependent addressing. Argon2d, + /// Argon2i data-independent addressing. Argon2i, + /// Argon2id hybrid addressing. Argon2id, } @@ -223,6 +218,9 @@ pub enum Argon2Error { ResourceOverflow, /// The allocator refused to provide the memory matrix. AllocationFailed, + /// A forced diagnostic backend is unavailable on the current host. + #[cfg(feature = "diag")] + BackendUnavailable, /// Password generation parameters exceed the verifier's resource limits. #[cfg(feature = "phc-strings")] VerificationLimitTooLow, @@ -242,6 +240,8 @@ impl fmt::Display for Argon2Error { Self::AssociatedDataTooLong => "Argon2 associated data exceeds 2^32-1 bytes", Self::ResourceOverflow => "Argon2 memory matrix exceeds the target's address space", Self::AllocationFailed => "Argon2 memory-matrix allocation failed", + #[cfg(feature = "diag")] + Self::BackendUnavailable => "requested Argon2 diagnostic backend is unavailable", #[cfg(all(feature = "phc-strings", feature = "getrandom"))] Self::EntropyUnavailable => "Argon2 entropy source unavailable", #[cfg(feature = "phc-strings")] @@ -392,7 +392,11 @@ struct Argon2Shape { const fn argon2_shape(params: Argon2Params) -> Argon2Shape { let lane_group = params.parallelism.strict_mul(SYNC_POINTS); - let blocks = (params.memory_cost_kib / lane_group).strict_mul(lane_group); + let blocks = params + .memory_cost_kib + .checked_div(lane_group) + .expect("validated Argon2 parallelism makes the lane group nonzero") + .strict_mul(lane_group); Argon2Shape { blocks, memory_bytes: (blocks as u64).strict_mul(BLOCK_SIZE as u64), @@ -488,6 +492,24 @@ pub fn diag_hash_portable( ) } +#[cfg(all( + feature = "diag", + any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64", + target_arch = "wasm32", + ) +))] +fn diag_compress_for(kernel: KernelId) -> Result { + if crate::platform::caps().has(dispatch::required_caps(kernel)) { + Ok(dispatch::compress_fn_for(kernel)) + } else { + Err(Argon2Error::BackendUnavailable) + } +} + /// Hash via the aarch64 NEON kernel. /// /// # Errors @@ -515,13 +537,7 @@ pub fn diag_hash_aarch64_neon( /// /// # Errors /// -/// Returns [`Argon2Error`] for invalid parameters. -/// -/// # Panics -/// -/// Panics if the host does not support AVX2. The per-kernel tests -/// gate this call on `crate::platform::caps()` having the kernel's -/// required caps before invoking it. +/// Returns [`Argon2Error`] for invalid parameters or when AVX2 is unavailable. #[cfg(all(feature = "diag", target_arch = "x86_64"))] pub fn diag_hash_x86_avx2( params: &Argon2Params, @@ -530,17 +546,13 @@ pub fn diag_hash_x86_avx2( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - assert!( - crate::platform::caps().has(dispatch::required_caps(KernelId::X86Avx2)), - "AVX2 not available on host" - ); argon2_hash_with_kernel( params, password, salt, variant, out, - dispatch::compress_fn_for(KernelId::X86Avx2), + diag_compress_for(KernelId::X86Avx2)?, ) } @@ -548,11 +560,7 @@ pub fn diag_hash_x86_avx2( /// /// # Errors /// -/// Returns [`Argon2Error`] for invalid parameters. -/// -/// # Panics -/// -/// Panics if the host does not support AVX-512F + AVX-512VL. +/// Returns [`Argon2Error`] for invalid parameters or when AVX-512F plus AVX-512VL is unavailable. #[cfg(all(feature = "diag", target_arch = "x86_64"))] pub fn diag_hash_x86_avx512( params: &Argon2Params, @@ -561,17 +569,13 @@ pub fn diag_hash_x86_avx512( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - assert!( - crate::platform::caps().has(dispatch::required_caps(KernelId::X86Avx512)), - "AVX-512F + AVX-512VL not available on host" - ); argon2_hash_with_kernel( params, password, salt, variant, out, - dispatch::compress_fn_for(KernelId::X86Avx512), + diag_compress_for(KernelId::X86Avx512)?, ) } @@ -579,11 +583,7 @@ pub fn diag_hash_x86_avx512( /// /// # Errors /// -/// Returns [`Argon2Error`] for invalid parameters. -/// -/// # Panics -/// -/// Panics if the host does not support VSX. +/// Returns [`Argon2Error`] for invalid parameters or when VSX is unavailable. #[cfg(all(feature = "diag", target_arch = "powerpc64"))] pub fn diag_hash_power_vsx( params: &Argon2Params, @@ -592,17 +592,13 @@ pub fn diag_hash_power_vsx( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - assert!( - crate::platform::caps().has(dispatch::required_caps(KernelId::PowerVsx)), - "POWER VSX not available on host" - ); argon2_hash_with_kernel( params, password, salt, variant, out, - dispatch::compress_fn_for(KernelId::PowerVsx), + diag_compress_for(KernelId::PowerVsx)?, ) } @@ -610,11 +606,7 @@ pub fn diag_hash_power_vsx( /// /// # Errors /// -/// Returns [`Argon2Error`] for invalid parameters. -/// -/// # Panics -/// -/// Panics if the host does not support the z13+ vector facility. +/// Returns [`Argon2Error`] for invalid parameters or when the z13+ vector facility is unavailable. #[cfg(all(feature = "diag", target_arch = "s390x"))] pub fn diag_hash_s390x_vector( params: &Argon2Params, @@ -623,17 +615,13 @@ pub fn diag_hash_s390x_vector( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - assert!( - crate::platform::caps().has(dispatch::required_caps(KernelId::S390xVector)), - "s390x vector facility not available on host" - ); argon2_hash_with_kernel( params, password, salt, variant, out, - dispatch::compress_fn_for(KernelId::S390xVector), + diag_compress_for(KernelId::S390xVector)?, ) } @@ -641,11 +629,7 @@ pub fn diag_hash_s390x_vector( /// /// # Errors /// -/// Returns [`Argon2Error`] for invalid parameters. -/// -/// # Panics -/// -/// Panics if the host does not support the RISC-V V extension. +/// Returns [`Argon2Error`] for invalid parameters or when the RISC-V V extension is unavailable. #[cfg(all(feature = "diag", target_arch = "riscv64"))] pub fn diag_hash_riscv64_v( params: &Argon2Params, @@ -654,17 +638,13 @@ pub fn diag_hash_riscv64_v( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - assert!( - crate::platform::caps().has(dispatch::required_caps(KernelId::Riscv64V)), - "RISC-V V extension not available on host" - ); argon2_hash_with_kernel( params, password, salt, variant, out, - dispatch::compress_fn_for(KernelId::Riscv64V), + diag_compress_for(KernelId::Riscv64V)?, ) } @@ -672,11 +652,7 @@ pub fn diag_hash_riscv64_v( /// /// # Errors /// -/// Returns [`Argon2Error`] for invalid parameters. -/// -/// # Panics -/// -/// Panics if the host does not support wasm SIMD128. +/// Returns [`Argon2Error`] for invalid parameters or when WASM SIMD128 is unavailable. #[cfg(all(feature = "diag", target_arch = "wasm32"))] pub fn diag_hash_wasm_simd128( params: &Argon2Params, @@ -685,17 +661,13 @@ pub fn diag_hash_wasm_simd128( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - assert!( - crate::platform::caps().has(dispatch::required_caps(KernelId::WasmSimd128)), - "wasm simd128 not available on host" - ); argon2_hash_with_kernel( params, password, salt, variant, out, - dispatch::compress_fn_for(KernelId::WasmSimd128), + diag_compress_for(KernelId::WasmSimd128)?, ) } @@ -866,7 +838,7 @@ fn h_prime(input_parts: &[&[u8]], out: &mut [u8]) { // Feed LE32(out_len) then the input parts into Blake2b. For out_len <= 64 // the single-block output is the answer directly. let len_le = u32::try_from(out_len) - .unwrap_or_else(|_| unreachable!("Argon2 H' output length was checked before expansion")) + .expect("Argon2 output length is validated before H' expansion") .to_le_bytes(); if out_len <= 64 { @@ -916,13 +888,17 @@ fn h_prime_diag_blake2b_portable(input_parts: &[&[u8]], out: &mut [u8]) { let out_len = out.len(); assert!(out_len > 0, "H' output length must be positive"); let len_le = u32::try_from(out_len) - .unwrap_or_else(|_| unreachable!("Argon2 H' output length was checked before expansion")) + .expect("Argon2 output length is validated before H' expansion") .to_le_bytes(); if out_len <= 64 { let parts = [&len_le[..], input_parts[0]]; if input_parts.len() == 1 { - crate::hashes::crypto::blake2b::diag_hash_parts_portable(out_len as u8, &parts, out); + crate::hashes::crypto::blake2b::diag_hash_parts_portable( + u8::try_from(out_len).expect("the single-digest branch limits output to 64 bytes"), + &parts, + out, + ); } else { let mut data = [0u8; BLOCK_SIZE + 16]; let mut pos = 0usize; @@ -932,7 +908,11 @@ fn h_prime_diag_blake2b_portable(input_parts: &[&[u8]], out: &mut [u8]) { data[pos..pos.strict_add(part.len())].copy_from_slice(part); pos = pos.strict_add(part.len()); } - crate::hashes::crypto::blake2b::diag_hash_parts_portable(out_len as u8, &[&data[..pos]], out); + crate::hashes::crypto::blake2b::diag_hash_parts_portable( + u8::try_from(out_len).expect("the single-digest branch limits output to 64 bytes"), + &[&data[..pos]], + out, + ); } return; } @@ -960,7 +940,11 @@ fn h_prime_diag_blake2b_portable(input_parts: &[&[u8]], out: &mut [u8]) { let tail_off = r.strict_mul(32); let tail_len = out_len.strict_sub(tail_off); - crate::hashes::crypto::blake2b::diag_hash_parts_portable(tail_len as u8, &[&v_prev], &mut out[tail_off..]); + crate::hashes::crypto::blake2b::diag_hash_parts_portable( + u8::try_from(tail_len).expect("the H' tail is at most 64 bytes"), + &[&v_prev], + &mut out[tail_off..], + ); ct::zeroize(&mut v_prev); } @@ -979,26 +963,26 @@ fn compute_h0( // `check_inputs` before reaching here, so the `try_from` calls below are // infallible. We use `try_from + expect` rather than `as u32` to keep the // invariant readable at the call site. - let len_u32 = |label: &'static str, len: usize| -> [u8; 4] { + let len_u32 = |len: usize| -> [u8; 4] { u32::try_from(len) - .unwrap_or_else(|_| panic!("Argon2 H0: {label} length exceeded MAX_VAR_BYTES; check_inputs should have rejected")) + .expect("Argon2 input lengths are validated before H0 construction") .to_le_bytes() }; let mut hasher = Blake2b512::new(); hasher.update(¶ms.parallelism.to_le_bytes()); - hasher.update(&len_u32("output", output_len)); + hasher.update(&len_u32(output_len)); hasher.update(¶ms.memory_cost_kib.to_le_bytes()); hasher.update(¶ms.time_cost.to_le_bytes()); hasher.update(&ARGON2_VERSION.to_le_bytes()); hasher.update(&variant.y().to_le_bytes()); - hasher.update(&len_u32("password", password.len())); + hasher.update(&len_u32(password.len())); hasher.update(password); - hasher.update(&len_u32("salt", salt.len())); + hasher.update(&len_u32(salt.len())); hasher.update(salt); - hasher.update(&len_u32("secret", context.secret.len())); + hasher.update(&len_u32(context.secret.len())); hasher.update(context.secret); - hasher.update(&len_u32("associated_data", context.associated_data.len())); + hasher.update(&len_u32(context.associated_data.len())); hasher.update(context.associated_data); hasher.finalize() } @@ -1012,22 +996,22 @@ fn compute_h0_diag_blake2b_portable( variant: Argon2Variant, output_len: usize, ) -> [u8; 64] { - let len_u32 = |label: &'static str, len: usize| -> [u8; 4] { + let len_u32 = |len: usize| -> [u8; 4] { u32::try_from(len) - .unwrap_or_else(|_| panic!("Argon2 H0: {label} length exceeded MAX_VAR_BYTES; check_inputs should have rejected")) + .expect("Argon2 input lengths are validated before H0 construction") .to_le_bytes() }; let parallelism = params.parallelism.to_le_bytes(); - let output_len = len_u32("output", output_len); + let output_len = len_u32(output_len); let memory_cost = params.memory_cost_kib.to_le_bytes(); let time_cost = params.time_cost.to_le_bytes(); let version = ARGON2_VERSION.to_le_bytes(); let variant = variant.y().to_le_bytes(); - let password_len = len_u32("password", password.len()); - let salt_len = len_u32("salt", salt.len()); - let secret_len = len_u32("secret", context.secret.len()); - let associated_data_len = len_u32("associated_data", context.associated_data.len()); + let password_len = len_u32(password.len()); + let salt_len = len_u32(salt.len()); + let secret_len = len_u32(context.secret.len()); + let associated_data_len = len_u32(context.associated_data.len()); let mut out = [0u8; 64]; crate::hashes::crypto::blake2b::diag_hash_parts_portable( 64, @@ -1057,12 +1041,8 @@ fn compute_h0_diag_blake2b_portable( #[inline(always)] fn block_from_bytes(bytes: &[u8; BLOCK_SIZE]) -> MemoryBlock { let mut out = MemoryBlock::zero(); - for i in 0..BLOCK_WORDS { - out.0[i] = u64::from_le_bytes( - bytes[i.strict_mul(8)..i.strict_mul(8).strict_add(8)] - .try_into() - .unwrap(), - ); + for (word, bytes) in out.0.iter_mut().zip(bytes.as_chunks::<8>().0) { + *word = u64::from_le_bytes(*bytes); } out } @@ -1078,6 +1058,19 @@ fn block_to_bytes(block: &[u64; BLOCK_WORDS]) -> [u8; BLOCK_SIZE] { // ─── Argon2i pseudo-random address stream (RFC 9106 §3.4.2) ──────────────── +#[derive(Clone, Copy)] +struct SegmentConfig { + compress: CompressFn, + pass: u32, + slice: u32, + lanes: u32, + segment_len: u32, + lane_len: u32, + total_blocks: u32, + variant: Argon2Variant, + time_cost: u32, +} + /// Buffer of 128 `J1||J2` word pairs used for a single segment of Argon2i /// (or Argon2id's data-independent slices). #[derive(Clone)] @@ -1094,25 +1087,14 @@ impl AddressBlock { /// Generate a fresh address block keyed by /// `(pass, lane, slice, blocks, total_passes, variant_y, counter)`. - #[allow(clippy::too_many_arguments)] // RFC 9106 §3.4.2 fixes this list; wrapping it in a struct is empty ceremony. - fn refresh( - &mut self, - compress: CompressFn, - pass: u32, - lane: u32, - slice: u32, - blocks: u32, - total_passes: u32, - variant_y: u32, - counter: u64, - ) { + fn refresh(&mut self, config: SegmentConfig, lane: u32, counter: u64) { let mut input = MemoryBlock::zero(); - input.0[0] = pass as u64; - input.0[1] = lane as u64; - input.0[2] = slice as u64; - input.0[3] = blocks as u64; - input.0[4] = total_passes as u64; - input.0[5] = variant_y as u64; + input.0[0] = u64::from(config.pass); + input.0[1] = u64::from(lane); + input.0[2] = u64::from(config.slice); + input.0[3] = u64::from(config.total_blocks); + input.0[4] = u64::from(config.time_cost); + input.0[5] = u64::from(config.variant.y()); input.0[6] = counter; let zero = MemoryBlock::zero(); @@ -1121,8 +1103,8 @@ impl AddressBlock { // `compress_fn_for` in per-kernel tests), which only returns a // kernel whose `required_caps` are a subset of the host's caps. unsafe { - compress(&mut intermediate.0, &zero.0, &input.0, /* xor_into = */ false); - compress(&mut self.words.0, &zero.0, &intermediate.0, /* xor_into = */ false); + (config.compress)(&mut intermediate.0, &zero.0, &input.0, /* xor_into = */ false); + (config.compress)(&mut self.words.0, &zero.0, &intermediate.0, /* xor_into = */ false); } } } @@ -1143,12 +1125,16 @@ impl Matrix { let shape = argon2_shape(params); let lanes = params.parallelism; let m_prime = shape.blocks; - let lane_len = m_prime / lanes; - let segment_len = lane_len / SYNC_POINTS; + let lane_len = m_prime + .checked_div(lanes) + .expect("validated Argon2 parallelism is nonzero"); + let segment_len = lane_len + .checked_div(SYNC_POINTS) + .expect("the Argon2 synchronization-point count is nonzero"); if shape.memory_bytes > isize::MAX as u64 { return Err(Argon2Error::ResourceOverflow); } - let total = m_prime as usize; + let total = usize::try_from(m_prime).map_err(|_| Argon2Error::ResourceOverflow)?; let mut blocks = Vec::new(); blocks .try_reserve_exact(total) @@ -1286,27 +1272,21 @@ impl MatrixView { /// Compute the reference block `(ref_lane, ref_index)` for a position /// `(lane, col)` in pass `pass`, using `j1`/`j2` pseudo-random words. -#[allow(clippy::too_many_arguments)] // RFC 9106 §3.4 ties eight fields together; a ctx struct would just forward them all. #[inline(always)] -fn reference_index( - pass: u32, - lane: u32, - slice: u32, - col: u32, - j1: u32, - j2: u32, - lanes: u32, - segment_len: u32, - lane_len: u32, -) -> (u32, u32) { +fn reference_index(config: SegmentConfig, lane: u32, col: u32, j1: u32, j2: u32) -> (u32, u32) { // Reference lane - let ref_lane = if pass == 0 && slice == 0 { lane } else { j2 % lanes }; + let ref_lane = if config.pass == 0 && config.slice == 0 { + lane + } else { + j2.checked_rem(config.lanes) + .expect("validated Argon2 parallelism is nonzero") + }; // Reference area size let same_lane = ref_lane == lane; - let position_in_segment = col.wrapping_sub(slice.wrapping_mul(segment_len)); + let position_in_segment = col.wrapping_sub(config.slice.wrapping_mul(config.segment_len)); // We know position_in_segment < segment_len (by construction). - let area_size: u32 = if pass == 0 { + let area_size: u32 = if config.pass == 0 { // First pass: previous slices in the current lane are available. if same_lane { // Blocks 0..col − 1 available (col excludes position itself). @@ -1316,7 +1296,7 @@ fn reference_index( } else { // Other lane: blocks in slices 0..slice completed, minus 1 if position // in current segment == 0 (prevents self-reference in racing lane). - let completed_slices = slice.wrapping_mul(segment_len); + let completed_slices = config.slice.wrapping_mul(config.segment_len); if position_in_segment == 0 { completed_slices.wrapping_sub(1) } else { @@ -1328,13 +1308,14 @@ fn reference_index( // already-computed portion for same-lane references. if same_lane { // lane_len − segment_len + position_in_segment − 1 - lane_len - .wrapping_sub(segment_len) + config + .lane_len + .wrapping_sub(config.segment_len) .wrapping_add(position_in_segment) .wrapping_sub(1) } else { // lane_len − segment_len, minus 1 if position_in_segment == 0 - let base = lane_len.wrapping_sub(segment_len); + let base = config.lane_len.wrapping_sub(config.segment_len); if position_in_segment == 0 { base.wrapping_sub(1) } else { @@ -1347,18 +1328,23 @@ fn reference_index( let j1_u64 = j1 as u64; let relative_position = { let x = (j1_u64.wrapping_mul(j1_u64)) >> 32; - let y = (area_size as u64).wrapping_mul(x) >> 32; - (area_size as u64).wrapping_sub(1).wrapping_sub(y) as u32 + let y = u64::from(area_size).wrapping_mul(x) >> 32; + let relative = u64::from(area_size).wrapping_sub(1).wrapping_sub(y); + let [b0, b1, b2, b3, _, _, _, _] = relative.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) }; // Absolute start position of the reference area (wraps across the lane). - let start_position = if pass == 0 || slice == (SYNC_POINTS - 1) { + let start_position = if config.pass == 0 || config.slice == (SYNC_POINTS - 1) { 0 } else { - (slice.wrapping_add(1)).wrapping_mul(segment_len) + (config.slice.wrapping_add(1)).wrapping_mul(config.segment_len) }; - let ref_index = (start_position.wrapping_add(relative_position)) % lane_len; + let ref_index = start_position + .wrapping_add(relative_position) + .checked_rem(config.lane_len) + .expect("validated Argon2 geometry has a nonzero lane length"); (ref_lane, ref_index) } @@ -1369,7 +1355,6 @@ fn reference_index( /// [`MatrixView`] from the underlying block storage, and dispatches into /// [`fill_segment_inner`]. The exclusive borrow makes the inner kernel's /// safety contract trivially satisfied for single-threaded callers. -#[allow(clippy::too_many_arguments)] // RFC 9106 §3.4 fixes this list; struct wrapper is empty ceremony. fn fill_segment( matrix: &mut Matrix, compress: CompressFn, @@ -1379,29 +1364,24 @@ fn fill_segment( variant: Argon2Variant, time_cost: u32, ) { - let lanes = matrix.lanes; - let segment_len = matrix.segment_len; - let lane_len = matrix.lane_len; - let total_blocks = matrix.len() as u32; + let config = SegmentConfig { + compress, + pass, + slice, + lanes: matrix.lanes, + segment_len: matrix.segment_len, + lane_len: matrix.lane_len, + total_blocks: u32::try_from(matrix.len()).expect("matrix length originates from the validated u32 block count"), + variant, + time_cost, + }; let view = MatrixView::from_blocks(&mut matrix.blocks); // SAFETY: `&mut matrix` is held exclusively for the duration of the // call. The view is the only handle to the matrix's storage during // `fill_segment_inner`; aliasing and concurrency contracts are // trivially upheld. unsafe { - fill_segment_inner( - view, - compress, - pass, - lane, - slice, - lanes, - segment_len, - lane_len, - total_blocks, - variant, - time_cost, - ); + fill_segment_inner(view, lane, config); } } @@ -1416,31 +1396,26 @@ fn fill_segment( /// /// Callers must guarantee that, for the indices touched by this call: /// -/// - The current segment range `[lane * lane_len + slice * segment_len .. lane * lane_len + (slice -/// + 1) * segment_len]` is exclusively writeable by this task. Same-task reads may target -/// already-written positions in that range. -/// - Every immutable input remains stable for the duration of `compress`. The previous and -/// reference inputs may alias each other, but neither may alias the current mutable output. +/// - The task has exclusive write access to the current lane and slice's segment range; same-task reads may target positions it has already written there. +/// - Every immutable input remains stable for `compress`; previous and reference inputs may alias each other, but neither may alias the current mutable output. /// /// Both conditions are upheld by: +/// /// - The sequential path's exclusive `&mut Matrix` borrow, OR /// - The Argon2 reference-index function (RFC 9106 §3.4) when called from `fill_slice_parallel` — /// see the doc-comment on [`MatrixView`] for the disjointness argument. -#[allow(clippy::too_many_arguments, clippy::doc_lazy_continuation)] -unsafe fn fill_segment_inner( - view: MatrixView, - compress: CompressFn, - pass: u32, - lane: u32, - slice: u32, - lanes: u32, - segment_len: u32, - lane_len: u32, - total_blocks: u32, - variant: Argon2Variant, - time_cost: u32, -) { - let variant_y = variant.y(); +unsafe fn fill_segment_inner(view: MatrixView, lane: u32, config: SegmentConfig) { + let SegmentConfig { + compress, + pass, + slice, + lanes: _, + segment_len, + lane_len, + total_blocks: _, + variant, + time_cost: _, + } = config; let lane_len_usize = lane_len as usize; let lane_base = (lane as usize).strict_mul(lane_len_usize); @@ -1458,16 +1433,7 @@ unsafe fn fill_segment_inner( let mut address_counter: u64 = 0; if is_independent { address_counter = 1; // RFC 9106: counter starts at 1 for first block. - address_block.refresh( - compress, - pass, - lane, - slice, - total_blocks, - time_cost, - variant_y, - address_counter, - ); + address_block.refresh(config, lane, address_counter); } // Starting column for this segment. Skip first two blocks of lane 0 on @@ -1491,16 +1457,7 @@ unsafe fn fill_segment_inner( let addr_pos = (seg_col as usize) % BLOCK_WORDS; if addr_pos == 0 && seg_col != 0 { address_counter = address_counter.strict_add(1); - address_block.refresh( - compress, - pass, - lane, - slice, - total_blocks, - time_cost, - variant_y, - address_counter, - ); + address_block.refresh(config, lane, address_counter); } let word = address_block.words.0[addr_pos]; ((word & 0xFFFF_FFFFu64) as u32, (word >> 32) as u32) @@ -1515,7 +1472,7 @@ unsafe fn fill_segment_inner( ((word & 0xFFFF_FFFFu64) as u32, (word >> 32) as u32) }; - let (ref_lane, ref_index) = reference_index(pass, lane, slice, col, j1, j2, lanes, segment_len, lane_len); + let (ref_lane, ref_index) = reference_index(config, lane, col, j1, j2); // Compute new block: G(B[lane][prev], B[ref_lane][ref_index]), optionally // XOR-accumulated into existing block (v1.3, pass > 0). @@ -1591,8 +1548,19 @@ fn fill_slice_parallel( let lanes = matrix.lanes; let segment_len = matrix.segment_len; let lane_len = matrix.lane_len; - let total_blocks = matrix.len() as u32; + let total_blocks = u32::try_from(matrix.len()).expect("matrix length originates from the validated u32 block count"); let view = MatrixView::from_blocks(&mut matrix.blocks); + let config = SegmentConfig { + compress, + pass, + slice, + lanes, + segment_len, + lane_len, + total_blocks, + variant, + time_cost, + }; // Spawn lanes 1..lanes onto rayon workers; run lane 0 inline on the // calling thread. This is the standard Blake3 / parallel-tree-reduction @@ -1617,19 +1585,7 @@ fn fill_slice_parallel( // a writer. The Send/Sync impls on `MatrixView` are sound under // this discipline; see the `MatrixView` doc-comment. unsafe { - fill_segment_inner( - view, - compress, - pass, - lane, - slice, - lanes, - segment_len, - lane_len, - total_blocks, - variant, - time_cost, - ); + fill_segment_inner(view, lane, config); } }); } @@ -1638,19 +1594,7 @@ fn fill_slice_parallel( // call (no other spawned task touches lane 0). The same disjointness // argument used for the spawned tasks applies — see above. unsafe { - fill_segment_inner( - view, - compress, - pass, - 0, - slice, - lanes, - segment_len, - lane_len, - total_blocks, - variant, - time_cost, - ); + fill_segment_inner(view, 0, config); } }); } @@ -1678,6 +1622,13 @@ fn fill_slice( // ─── Full Argon2 hash function ───────────────────────────────────────────── +#[derive(Clone, Copy)] +struct HashBackend { + compress: CompressFn, + #[cfg(feature = "diag")] + diag_blake2b: bool, +} + fn argon2_hash( params: &Argon2Params, password: &[u8], @@ -1696,7 +1647,19 @@ fn argon2_hash_with_context( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - argon2_hash_with_kernel_inner(params, context, password, salt, variant, out, active_compress(), false) + argon2_hash_with_kernel_inner( + params, + context, + password, + salt, + variant, + out, + HashBackend { + compress: active_compress(), + #[cfg(feature = "diag")] + diag_blake2b: false, + }, + ) } #[cfg(feature = "diag")] @@ -1715,8 +1678,10 @@ fn argon2_hash_with_kernel( salt, variant, out, - compress, - false, + HashBackend { + compress, + diag_blake2b: false, + }, ) } @@ -1736,12 +1701,13 @@ fn argon2_hash_with_kernel_diag_blake2b( salt, variant, out, - compress, - true, + HashBackend { + compress, + diag_blake2b: true, + }, ) } -#[allow(clippy::too_many_arguments)] // Params, context, inputs, variant, output, and kernel are the real operation boundary. fn argon2_hash_with_kernel_inner( params: &Argon2Params, context: Argon2Context<'_>, @@ -1749,8 +1715,7 @@ fn argon2_hash_with_kernel_inner( salt: &[u8], variant: Argon2Variant, out: &mut [u8], - compress: CompressFn, - #[cfg_attr(not(feature = "diag"), allow(unused_variables))] diag_blake2b: bool, + backend: HashBackend, ) -> Result<(), Argon2Error> { Argon2Params::check_inputs(password, salt, context)?; if out.len() < MIN_OUTPUT_LEN || out.len() as u64 > MAX_VAR_BYTES { @@ -1767,7 +1732,7 @@ fn argon2_hash_with_kernel_inner( let mut h0 = { #[cfg(feature = "diag")] { - if diag_blake2b { + if backend.diag_blake2b { compute_h0_diag_blake2b_portable(params, context, password, salt, variant, out.len()) } else { compute_h0(params, context, password, salt, variant, out.len()) @@ -1785,7 +1750,7 @@ fn argon2_hash_with_kernel_inner( // B[lane][0] = H'(H0 || LE32(0) || LE32(lane), BLOCK_SIZE) let lane_le = lane.to_le_bytes(); #[cfg(feature = "diag")] - if diag_blake2b { + if backend.diag_blake2b { h_prime_diag_blake2b_portable(&[&h0, &0u32.to_le_bytes(), &lane_le], &mut buf); } else { h_prime(&[&h0, &0u32.to_le_bytes(), &lane_le], &mut buf); @@ -1796,7 +1761,7 @@ fn argon2_hash_with_kernel_inner( // B[lane][1] = H'(H0 || LE32(1) || LE32(lane), BLOCK_SIZE) #[cfg(feature = "diag")] - if diag_blake2b { + if backend.diag_blake2b { h_prime_diag_blake2b_portable(&[&h0, &1u32.to_le_bytes(), &lane_le], &mut buf); } else { h_prime(&[&h0, &1u32.to_le_bytes(), &lane_le], &mut buf); @@ -1812,21 +1777,22 @@ fn argon2_hash_with_kernel_inner( // skipped when `parallelism == 1` to avoid rayon overhead. for pass in 0..params.time_cost { for slice in 0..SYNC_POINTS { - fill_slice(&mut matrix, compress, pass, slice, variant, params.time_cost); + fill_slice(&mut matrix, backend.compress, pass, slice, variant, params.time_cost); } } // Finalisation: C = XOR of last block of each lane; output = H'(C, T). - let mut acc = *matrix.get(0, lane_len - 1); + let last_block = lane_len.strict_sub(1); + let mut acc = *matrix.get(0, last_block); for lane in 1..lanes { - let blk = matrix.get(lane, lane_len - 1); + let blk = matrix.get(lane, last_block); for i in 0..BLOCK_WORDS { acc[i] ^= blk[i]; } } let mut acc_bytes = block_to_bytes(&acc); #[cfg(feature = "diag")] - if diag_blake2b { + if backend.diag_blake2b { h_prime_diag_blake2b_portable(&[&acc_bytes], out); } else { h_prime(&[&acc_bytes], out); @@ -2148,7 +2114,7 @@ mod password_phc { } impl ApprovedPhc { - pub fn salt(&self) -> &[u8] { + pub(super) fn salt(&self) -> &[u8] { &self.salt[..self.salt_len as usize] } } @@ -2209,7 +2175,7 @@ mod password_phc { Ok(ApprovedPhc { params, salt, - salt_len: decoded_salt_len as u8, + salt_len: u8::try_from(decoded_salt_len).map_err(|_| PhcError::InvalidLength)?, expected, }) } @@ -2252,7 +2218,7 @@ mod tests { #[cfg(not(miri))] fn canon_params() -> Argon2Params { - Argon2Params::new(32, 3, 4).unwrap() + Argon2Params::new(32, 3, 4).expect("RFC 9106 test parameters must be valid") } #[cfg(not(miri))] @@ -2277,11 +2243,14 @@ mod tests { ]; let mut actual = [0u8; 32]; - Argon2d::derive_with_context(&canon_params(), canon_context(), PASSWORD, SALT, &mut actual).unwrap(); + Argon2d::derive_with_context(&canon_params(), canon_context(), PASSWORD, SALT, &mut actual) + .expect("RFC 9106 Argon2d vector inputs must derive"); assert_eq!(actual, expected_d); - Argon2i::derive_with_context(&canon_params(), canon_context(), PASSWORD, SALT, &mut actual).unwrap(); + Argon2i::derive_with_context(&canon_params(), canon_context(), PASSWORD, SALT, &mut actual) + .expect("RFC 9106 Argon2i vector inputs must derive"); assert_eq!(actual, expected_i); - Argon2id::derive_with_context(&canon_params(), canon_context(), PASSWORD, SALT, &mut actual).unwrap(); + Argon2id::derive_with_context(&canon_params(), canon_context(), PASSWORD, SALT, &mut actual) + .expect("RFC 9106 Argon2id vector inputs must derive"); assert_eq!(actual, expected_id); } @@ -2290,11 +2259,17 @@ mod tests { fn raw_verify_accepts_only_the_exact_inputs() { let params = canon_params(); let mut expected = [0u8; 32]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut expected).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut expected).expect("canonical inputs must derive"); - assert!(Argon2id::verify(¶ms, PASSWORD, SALT, &expected).is_ok()); - assert!(Argon2id::verify(¶ms, b"wrong", SALT, &expected).is_err()); - assert!(Argon2id::verify(¶ms, PASSWORD, &[0xff; 16], &expected).is_err()); + assert_eq!(Argon2id::verify(¶ms, PASSWORD, SALT, &expected), Ok(())); + assert_eq!( + Argon2id::verify(¶ms, b"wrong", SALT, &expected), + Err(VerificationError::new()) + ); + assert_eq!( + Argon2id::verify(¶ms, PASSWORD, &[0xff; 16], &expected), + Err(VerificationError::new()) + ); } #[test] @@ -2302,12 +2277,15 @@ mod tests { assert_eq!(Argon2Params::new(8, 0, 1), Err(Argon2Error::InvalidTimeCost)); assert_eq!(Argon2Params::new(8, 1, 0), Err(Argon2Error::InvalidParallelism)); assert_eq!(Argon2Params::new(16, 1, 4), Err(Argon2Error::InvalidMemoryCost)); - assert!(Argon2Params::new(32, 1, 4).is_ok()); + let valid = Argon2Params::new(32, 1, 4).expect("boundary-valid parameters must construct"); + assert_eq!(valid.get_memory_cost_kib(), 32); + assert_eq!(valid.get_time_cost(), 1); + assert_eq!(valid.get_parallelism(), 4); } #[test] fn derive_rejects_invalid_operation_lengths() { - let params = Argon2Params::new(32, 1, 4).unwrap(); + let params = Argon2Params::new(32, 1, 4).expect("test parameters must be valid"); let mut out = [0u8; 32]; assert_eq!( Argon2id::derive(¶ms, b"pw", &[0u8; 7], &mut out), @@ -2347,10 +2325,13 @@ mod tests { parallelism: u32, output_len: usize, ) -> vec::Vec { - let params = argon2::Params::new(memory_kib, time, parallelism, Some(output_len)).unwrap(); + let params = + argon2::Params::new(memory_kib, time, parallelism, Some(output_len)).expect("oracle parameters must be valid"); let oracle = argon2::Argon2::new(algorithm, argon2::Version::V0x13, params); let mut output = alloc::vec![0u8; output_len]; - oracle.hash_password_into(password, salt, &mut output).unwrap(); + oracle + .hash_password_into(password, salt, &mut output) + .expect("oracle inputs must derive"); output } @@ -2359,10 +2340,10 @@ mod tests { fn all_variants_match_the_oracle() { let cases: &[(u32, u32, u32, usize)] = &[(8, 1, 1, 16), (16, 2, 1, 32), (32, 3, 2, 64)]; for &(memory, time, parallelism, output_len) in cases { - let params = Argon2Params::new(memory, time, parallelism).unwrap(); + let params = Argon2Params::new(memory, time, parallelism).expect("oracle-case parameters must be valid"); let mut actual = alloc::vec![0u8; output_len]; - Argon2d::derive(¶ms, b"password", &[0u8; 16], &mut actual).unwrap(); + Argon2d::derive(¶ms, b"password", &[0u8; 16], &mut actual).expect("Argon2d oracle case must derive"); assert_eq!( actual, oracle_hash( @@ -2376,7 +2357,7 @@ mod tests { ) ); - Argon2i::derive(¶ms, b"password", &[0u8; 16], &mut actual).unwrap(); + Argon2i::derive(¶ms, b"password", &[0u8; 16], &mut actual).expect("Argon2i oracle case must derive"); assert_eq!( actual, oracle_hash( @@ -2390,7 +2371,7 @@ mod tests { ) ); - Argon2id::derive(¶ms, b"password", &[0u8; 16], &mut actual).unwrap(); + Argon2id::derive(¶ms, b"password", &[0u8; 16], &mut actual).expect("Argon2id oracle case must derive"); assert_eq!( actual, oracle_hash( @@ -2421,19 +2402,20 @@ mod tests { use crate::auth::{PasswordStatus, phc::PhcError}; fn small_params() -> Argon2Params { - Argon2Params::new(32, 2, 1).unwrap() + Argon2Params::new(32, 2, 1).expect("small test profile must be valid") } fn encode(params: Argon2Params, password: &[u8], salt: &[u8], context: Argon2Context<'_>) -> alloc::string::String { let mut verifier = [0u8; PASSWORD_OUTPUT_LEN]; - Argon2id::derive_with_context(¶ms, context, password, salt, &mut verifier).unwrap(); + Argon2id::derive_with_context(¶ms, context, password, salt, &mut verifier) + .expect("PHC test inputs must derive"); password_phc::encode(params, salt, &verifier) } #[test] fn canonical_password_record_round_trips() { let params = small_params(); - let password = Argon2idPassword::new(params).unwrap(); + let password = Argon2idPassword::new(params).expect("test verification profile must be admissible"); let encoded = encode(params, b"password", &[0xaa; 16], Argon2Context::default()); assert!(encoded.starts_with("$argon2id$v=19$m=32,t=2,p=1$")); @@ -2441,13 +2423,16 @@ mod tests { password.verify_password(b"password", &encoded), Ok(PasswordStatus::Current) ); - assert!(password.verify_password(b"wrong", &encoded).is_err()); + assert_eq!( + password.verify_password(b"wrong", &encoded), + Err(VerificationError::new()) + ); } #[test] fn accepted_older_profile_requests_rehash() { - let generation = Argon2Params::new(40, 2, 1).unwrap(); - let password = Argon2idPassword::new(generation).unwrap(); + let generation = Argon2Params::new(40, 2, 1).expect("generation profile must be valid"); + let password = Argon2idPassword::new(generation).expect("generation profile must be admissible"); let encoded = encode(small_params(), b"password", &[0xbb; 16], Argon2Context::default()); assert_eq!( @@ -2459,7 +2444,7 @@ mod tests { #[test] fn accepted_noncurrent_salt_length_requests_rehash() { let params = small_params(); - let password = Argon2idPassword::new(params).unwrap(); + let password = Argon2idPassword::new(params).expect("test verification profile must be admissible"); let encoded = encode(params, b"password", &[0xbb; 8], Argon2Context::default()); assert_eq!( @@ -2471,19 +2456,21 @@ mod tests { #[test] fn borrowed_context_is_required_for_context_bound_records() { let params = small_params(); - let password = Argon2idPassword::new(params).unwrap(); + let password = Argon2idPassword::new(params).expect("test verification profile must be admissible"); let context = Argon2Context::new(b"pepper", b"tenant"); let encoded = encode(params, b"password", &[0xcc; 16], context); - assert!(password.verify_password(b"password", &encoded).is_err()); + assert_eq!( + password.verify_password(b"password", &encoded), + Err(VerificationError::new()) + ); assert_eq!( password.verify_password_with_context(b"password", &encoded, context), Ok(PasswordStatus::Current) ); - assert!( - password - .verify_password_with_context(b"password", &encoded, Argon2Context::new(b"wrong", b"tenant"),) - .is_err() + assert_eq!( + password.verify_password_with_context(b"password", &encoded, Argon2Context::new(b"wrong", b"tenant"),), + Err(VerificationError::new()) ); } @@ -2506,19 +2493,19 @@ mod tests { #[test] fn actual_argon2_shape_defines_the_limit() { let limits = Argon2VerificationLimits::for_profile(small_params()); - let rounded_equivalent = Argon2Params::new(35, 2, 1).unwrap(); + let rounded_equivalent = Argon2Params::new(35, 2, 1).expect("rounded test profile must be valid"); assert!(limits.allows(rounded_equivalent)); - let next_matrix = Argon2Params::new(36, 2, 1).unwrap(); + let next_matrix = Argon2Params::new(36, 2, 1).expect("next-matrix test profile must be valid"); assert!(!limits.allows(next_matrix)); } #[test] fn generator_and_parser_share_the_full_argon2_parallelism_domain() { - let params = Argon2Params::new(2_048, 1, 256).unwrap(); + let params = Argon2Params::new(2_048, 1, 256).expect("maximum-domain test profile must be valid"); let encoded = password_phc::encode(params, &[0x44; 16], &[0u8; PASSWORD_OUTPUT_LEN]); let limits = Argon2VerificationLimits::for_profile(params); - assert!(password_phc::approve(&encoded, limits).is_ok()); + password_phc::approve(&encoded, limits).expect("generator output must be accepted by the parser"); } #[test] @@ -2535,16 +2522,22 @@ mod tests { format!("$argon2id$v=19$m=32,t=2,x=1$${salt}$${hash}"), ]; for encoded in cases { - assert!(password_phc::approve(&encoded, limits).is_err(), "{encoded}"); + password_phc::approve(&encoded, limits) + .err() + .expect("noncanonical record must be rejected"); } } #[cfg(feature = "getrandom")] #[test] fn generated_records_use_fresh_salts() { - let password = Argon2idPassword::new(small_params()).unwrap(); - let first = password.hash_password(b"password").unwrap(); - let second = password.hash_password(b"password").unwrap(); + let password = Argon2idPassword::new(small_params()).expect("test verification profile must be admissible"); + let first = password + .hash_password(b"password") + .expect("first salt generation must succeed"); + let second = password + .hash_password(b"password") + .expect("second salt generation must succeed"); assert_ne!(first, second); assert_eq!( diff --git a/src/auth/argon2/power.rs b/src/auth/argon2/power.rs index 8ff495f0..4e96ea7d 100644 --- a/src/auth/argon2/power.rs +++ b/src/auth/argon2/power.rs @@ -26,12 +26,11 @@ //! //! # Rotations //! -//! Lane-wise u64 rotate is expressed as a shift-right + shift-left + OR -//! vector sequence. Whether the compiler selects a native rotate instruction -//! is a generated-code property, not a source contract. +//! Lane-wise u64 rotation uses `rotate_right`. Whether the compiler selects a +//! native vector rotate instruction is a generated-code property, not a source +//! contract. #![cfg(target_arch = "powerpc64")] -#![allow(clippy::cast_possible_truncation)] use core::simd::u64x2; @@ -53,94 +52,93 @@ pub(super) unsafe fn compress_vsx( y: &[u64; BLOCK_WORDS], xor_into: bool, ) { - // SAFETY: VSX is enabled by this function's `#[target_feature]`. - // `core::simd::u64x2` arithmetic is always wrapping per the SIMD - // contract, so no further unsafe is needed for the algorithm itself. let mut r = [0u64; BLOCK_WORDS]; let mut q = [0u64; BLOCK_WORDS]; - let mut i = 0; + let mut i = 0usize; while i < BLOCK_WORDS { - let xv = u64x2::from_array([x[i], x[i + 1]]); - let yv = u64x2::from_array([y[i], y[i + 1]]); + let next = i.strict_add(1); + let xv = u64x2::from_array([x[i], x[next]]); + let yv = u64x2::from_array([y[i], y[next]]); let rv = (xv ^ yv).to_array(); r[i] = rv[0]; - r[i + 1] = rv[1]; + r[next] = rv[1]; q[i] = rv[0]; - q[i + 1] = rv[1]; - i += 2; + q[next] = rv[1]; + i = i.strict_add(2); } // Row pass. let mut row = 0usize; while row < 8 { - let base = row * 16; - let mut a: Pair = [load_pair(&q, base), load_pair(&q, base + 2)]; - let mut b: Pair = [load_pair(&q, base + 4), load_pair(&q, base + 6)]; - let mut c: Pair = [load_pair(&q, base + 8), load_pair(&q, base + 10)]; - let mut d: Pair = [load_pair(&q, base + 12), load_pair(&q, base + 14)]; + let base = row.strict_mul(16); + let mut a: Pair = [load_pair(&q, base), load_pair(&q, base.strict_add(2))]; + let mut b: Pair = [load_pair(&q, base.strict_add(4)), load_pair(&q, base.strict_add(6))]; + let mut c: Pair = [load_pair(&q, base.strict_add(8)), load_pair(&q, base.strict_add(10))]; + let mut d: Pair = [load_pair(&q, base.strict_add(12)), load_pair(&q, base.strict_add(14))]; p_round(&mut a, &mut b, &mut c, &mut d); store_pair(&mut q, base, a[0]); - store_pair(&mut q, base + 2, a[1]); - store_pair(&mut q, base + 4, b[0]); - store_pair(&mut q, base + 6, b[1]); - store_pair(&mut q, base + 8, c[0]); - store_pair(&mut q, base + 10, c[1]); - store_pair(&mut q, base + 12, d[0]); - store_pair(&mut q, base + 14, d[1]); - row += 1; + store_pair(&mut q, base.strict_add(2), a[1]); + store_pair(&mut q, base.strict_add(4), b[0]); + store_pair(&mut q, base.strict_add(6), b[1]); + store_pair(&mut q, base.strict_add(8), c[0]); + store_pair(&mut q, base.strict_add(10), c[1]); + store_pair(&mut q, base.strict_add(12), d[0]); + store_pair(&mut q, base.strict_add(14), d[1]); + row = row.strict_add(1); } // Column pass. let mut col = 0usize; while col < 8 { - let base = col * 2; - let mut a: Pair = [load_pair(&q, base), load_pair(&q, base + 16)]; - let mut b: Pair = [load_pair(&q, base + 32), load_pair(&q, base + 48)]; - let mut c: Pair = [load_pair(&q, base + 64), load_pair(&q, base + 80)]; - let mut d: Pair = [load_pair(&q, base + 96), load_pair(&q, base + 112)]; + let base = col.strict_mul(2); + let mut a: Pair = [load_pair(&q, base), load_pair(&q, base.strict_add(16))]; + let mut b: Pair = [load_pair(&q, base.strict_add(32)), load_pair(&q, base.strict_add(48))]; + let mut c: Pair = [load_pair(&q, base.strict_add(64)), load_pair(&q, base.strict_add(80))]; + let mut d: Pair = [load_pair(&q, base.strict_add(96)), load_pair(&q, base.strict_add(112))]; p_round(&mut a, &mut b, &mut c, &mut d); store_pair(&mut q, base, a[0]); - store_pair(&mut q, base + 16, a[1]); - store_pair(&mut q, base + 32, b[0]); - store_pair(&mut q, base + 48, b[1]); - store_pair(&mut q, base + 64, c[0]); - store_pair(&mut q, base + 80, c[1]); - store_pair(&mut q, base + 96, d[0]); - store_pair(&mut q, base + 112, d[1]); - col += 1; + store_pair(&mut q, base.strict_add(16), a[1]); + store_pair(&mut q, base.strict_add(32), b[0]); + store_pair(&mut q, base.strict_add(48), b[1]); + store_pair(&mut q, base.strict_add(64), c[0]); + store_pair(&mut q, base.strict_add(80), c[1]); + store_pair(&mut q, base.strict_add(96), d[0]); + store_pair(&mut q, base.strict_add(112), d[1]); + col = col.strict_add(1); } // Final XOR with R, fused with dst store/xor. - let mut i = 0; + let mut i = 0usize; while i < BLOCK_WORDS { + let next = i.strict_add(1); let qv = load_pair(&q, i); let rv = load_pair(&r, i); let f = (qv ^ rv).to_array(); if xor_into { dst[i] ^= f[0]; - dst[i + 1] ^= f[1]; + dst[next] ^= f[1]; } else { dst[i] = f[0]; - dst[i + 1] = f[1]; + dst[next] = f[1]; } - i += 2; + i = i.strict_add(2); } } #[inline(always)] fn load_pair(buf: &[u64; BLOCK_WORDS], idx: usize) -> u64x2 { - u64x2::from_array([buf[idx], buf[idx + 1]]) + u64x2::from_array([buf[idx], buf[idx.strict_add(1)]]) } #[inline(always)] fn store_pair(buf: &mut [u64; BLOCK_WORDS], idx: usize, v: u64x2) { let a = v.to_array(); buf[idx] = a[0]; - buf[idx + 1] = a[1]; + buf[idx.strict_add(1)] = a[1]; } // ─── 4-way P-round ───────────────────────────────────────────────────────── @@ -201,32 +199,32 @@ fn gb(a: &mut Pair, b: &mut Pair, c: &mut Pair, d: &mut Pair) { // Step 1: a = a + b + 2·lsb(a)·lsb(b) let p0 = bla_mul(a[0], b[0]); let p1 = bla_mul(a[1], b[1]); - a[0] = a[0] + b[0] + p0; - a[1] = a[1] + b[1] + p1; + a[0] = bla_add(a[0], b[0], p0); + a[1] = bla_add(a[1], b[1], p1); d[0] = ror::<32>(d[0] ^ a[0]); d[1] = ror::<32>(d[1] ^ a[1]); // Step 2: c = c + d + 2·lsb(c)·lsb(d) let p0 = bla_mul(c[0], d[0]); let p1 = bla_mul(c[1], d[1]); - c[0] = c[0] + d[0] + p0; - c[1] = c[1] + d[1] + p1; + c[0] = bla_add(c[0], d[0], p0); + c[1] = bla_add(c[1], d[1], p1); b[0] = ror::<24>(b[0] ^ c[0]); b[1] = ror::<24>(b[1] ^ c[1]); // Step 3: a = a + b + 2·lsb(a)·lsb(b) let p0 = bla_mul(a[0], b[0]); let p1 = bla_mul(a[1], b[1]); - a[0] = a[0] + b[0] + p0; - a[1] = a[1] + b[1] + p1; + a[0] = bla_add(a[0], b[0], p0); + a[1] = bla_add(a[1], b[1], p1); d[0] = ror::<16>(d[0] ^ a[0]); d[1] = ror::<16>(d[1] ^ a[1]); // Step 4: c = c + d + 2·lsb(c)·lsb(d) let p0 = bla_mul(c[0], d[0]); let p1 = bla_mul(c[1], d[1]); - c[0] = c[0] + d[0] + p0; - c[1] = c[1] + d[1] + p1; + c[0] = bla_add(c[0], d[0], p0); + c[1] = bla_add(c[1], d[1], p1); b[0] = ror::<63>(b[0] ^ c[0]); b[1] = ror::<63>(b[1] ^ c[1]); } @@ -237,9 +235,14 @@ fn gb(a: &mut Pair, b: &mut Pair, c: &mut Pair, d: &mut Pair) { #[inline(always)] fn ror(v: u64x2) -> u64x2 { const { assert!(N > 0 && N < 64) } - let s = u64x2::splat(N as u64); - let r = u64x2::splat((64 - N) as u64); - (v >> s) | (v << r) + let right = core::ops::Shr::shr(v, u64x2::splat(u64::from(N))); + let left = core::ops::Shl::shl(v, u64x2::splat(u64::from(64u32.strict_sub(N)))); + core::ops::BitOr::bitor(right, left) +} + +#[inline(always)] +fn bla_add(a: u64x2, b: u64x2, product: u64x2) -> u64x2 { + simd_wrapping_add(simd_wrapping_add(a, b), product) } /// `2 · lsb(a) · lsb(b)` lane-wise via masked `u64x2 *`. The product @@ -248,7 +251,16 @@ fn ror(v: u64x2) -> u64x2 { #[inline(always)] fn bla_mul(a: u64x2, b: u64x2) -> u64x2 { let mask = u64x2::splat(0xffff_ffff); - let al = a & mask; - let bl = b & mask; - (al * bl) << u64x2::splat(1) + let product = simd_wrapping_mul(a & mask, b & mask); + core::ops::Shl::shl(product, u64x2::splat(1)) +} + +#[inline(always)] +fn simd_wrapping_add(a: u64x2, b: u64x2) -> u64x2 { + core::ops::Add::add(a, b) +} + +#[inline(always)] +fn simd_wrapping_mul(a: u64x2, b: u64x2) -> u64x2 { + core::ops::Mul::mul(a, b) } diff --git a/src/auth/argon2/riscv64.rs b/src/auth/argon2/riscv64.rs index 8e2255e6..c0205634 100644 --- a/src/auth/argon2/riscv64.rs +++ b/src/auth/argon2/riscv64.rs @@ -1,9 +1,8 @@ //! RISC-V Vector (RVV) BlaMka compression kernel for Argon2. //! -//! Uses two-u64 pairs to expose independent operations to the optimizer while -//! preserving ordinary scalar Rust semantics. Dispatch requires the RISC-V V -//! extension for this backend, but exact RVV lowering is target- and -//! toolchain-specific generated-code evidence. +//! Uses `core::simd::u64x2` pairs to expose two independent lanes per +//! operation. Dispatch requires the RISC-V V extension for this backend; +//! exact RVV lowering remains target- and toolchain-specific evidence. //! //! # Vectorisation topology //! @@ -14,29 +13,41 @@ //! ``` //! //! Column step: `GB(a_i, b_i, c_i, d_i)` for `i ∈ 0..4` runs at 4-way -//! parallelism with each `[u64; 2]` pair holding two of the four lanes +//! parallelism with each `u64x2` pair holding two of the four lanes //! per row (a/b/c/d × {lo, hi}). //! //! Diagonal step: rotate `b` by 1, `c` by 2, `d` by 3 within each -//! 4-lane row. Implemented as plain index swaps. +//! 4-lane row using cross-pair swizzles. //! //! # BlaMka multiply //! -//! `2 · lsb(a) · lsb(b)` lane-wise via masked `wrapping_mul`. The -//! product of two 32-bit values fits in u64, so masking with -//! `0xffffffff` and one `wrapping_mul(2)` is exact. +//! `2 · lsb(a) · lsb(b)` lane-wise via a masked SIMD multiply. The product of +//! two 32-bit values fits in u64, so the operation is exact modulo the lane +//! representation. //! //! # Rotations //! -//! All four (32, 24, 16, 63) use scalar `u64::rotate_right`; instruction -//! selection is left to the target compiler. +//! All four rotations use lane-wise SIMD shifts and OR. Instruction selection +//! is left to the target compiler. #![cfg(target_arch = "riscv64")] -#![allow(clippy::cast_possible_truncation)] + +use core::simd::u64x2; use super::BLOCK_WORDS; -type Pair = [u64; 2]; +type Pair = u64x2; + +struct RvvState { + a_lo: Pair, + a_hi: Pair, + b_lo: Pair, + b_hi: Pair, + c_lo: Pair, + c_hi: Pair, + d_lo: Pair, + d_hi: Pair, +} /// RISC-V V extension BlaMka compression kernel. /// @@ -53,225 +64,217 @@ pub(super) unsafe fn compress_rvv( y: &[u64; BLOCK_WORDS], xor_into: bool, ) { - // SAFETY: V extension is enabled by this function's `#[target_feature]`. - // Memory accesses are explicit indices into fixed-size arrays. let mut r = [0u64; BLOCK_WORDS]; let mut q = [0u64; BLOCK_WORDS]; - let mut i = 0; + let mut i = 0usize; while i < BLOCK_WORDS { - r[i] = x[i] ^ y[i]; - r[i + 1] = x[i + 1] ^ y[i + 1]; - q[i] = r[i]; - q[i + 1] = r[i + 1]; - i += 2; + let rv = vxor(load_pair(x, i), load_pair(y, i)); + store_pair(&mut r, i, rv); + store_pair(&mut q, i, rv); + i = i.strict_add(2); } // Row pass. let mut row = 0usize; while row < 8 { - let base = row * 16; - let mut a_lo: Pair = [q[base], q[base + 1]]; - let mut a_hi: Pair = [q[base + 2], q[base + 3]]; - let mut b_lo: Pair = [q[base + 4], q[base + 5]]; - let mut b_hi: Pair = [q[base + 6], q[base + 7]]; - let mut c_lo: Pair = [q[base + 8], q[base + 9]]; - let mut c_hi: Pair = [q[base + 10], q[base + 11]]; - let mut d_lo: Pair = [q[base + 12], q[base + 13]]; - let mut d_hi: Pair = [q[base + 14], q[base + 15]]; - - p_round( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - - q[base] = a_lo[0]; - q[base + 1] = a_lo[1]; - q[base + 2] = a_hi[0]; - q[base + 3] = a_hi[1]; - q[base + 4] = b_lo[0]; - q[base + 5] = b_lo[1]; - q[base + 6] = b_hi[0]; - q[base + 7] = b_hi[1]; - q[base + 8] = c_lo[0]; - q[base + 9] = c_lo[1]; - q[base + 10] = c_hi[0]; - q[base + 11] = c_hi[1]; - q[base + 12] = d_lo[0]; - q[base + 13] = d_lo[1]; - q[base + 14] = d_hi[0]; - q[base + 15] = d_hi[1]; - row += 1; + let base = row.strict_mul(16); + let mut state = RvvState { + a_lo: load_pair(&q, base), + a_hi: load_pair(&q, base.strict_add(2)), + b_lo: load_pair(&q, base.strict_add(4)), + b_hi: load_pair(&q, base.strict_add(6)), + c_lo: load_pair(&q, base.strict_add(8)), + c_hi: load_pair(&q, base.strict_add(10)), + d_lo: load_pair(&q, base.strict_add(12)), + d_hi: load_pair(&q, base.strict_add(14)), + }; + + p_round(&mut state); + + store_pair(&mut q, base, state.a_lo); + store_pair(&mut q, base.strict_add(2), state.a_hi); + store_pair(&mut q, base.strict_add(4), state.b_lo); + store_pair(&mut q, base.strict_add(6), state.b_hi); + store_pair(&mut q, base.strict_add(8), state.c_lo); + store_pair(&mut q, base.strict_add(10), state.c_hi); + store_pair(&mut q, base.strict_add(12), state.d_lo); + store_pair(&mut q, base.strict_add(14), state.d_hi); + row = row.strict_add(1); } // Column pass. let mut col = 0usize; while col < 8 { - let base = col * 2; - let mut a_lo: Pair = [q[base], q[base + 1]]; - let mut a_hi: Pair = [q[base + 16], q[base + 17]]; - let mut b_lo: Pair = [q[base + 32], q[base + 33]]; - let mut b_hi: Pair = [q[base + 48], q[base + 49]]; - let mut c_lo: Pair = [q[base + 64], q[base + 65]]; - let mut c_hi: Pair = [q[base + 80], q[base + 81]]; - let mut d_lo: Pair = [q[base + 96], q[base + 97]]; - let mut d_hi: Pair = [q[base + 112], q[base + 113]]; - - p_round( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - - q[base] = a_lo[0]; - q[base + 1] = a_lo[1]; - q[base + 16] = a_hi[0]; - q[base + 17] = a_hi[1]; - q[base + 32] = b_lo[0]; - q[base + 33] = b_lo[1]; - q[base + 48] = b_hi[0]; - q[base + 49] = b_hi[1]; - q[base + 64] = c_lo[0]; - q[base + 65] = c_lo[1]; - q[base + 80] = c_hi[0]; - q[base + 81] = c_hi[1]; - q[base + 96] = d_lo[0]; - q[base + 97] = d_lo[1]; - q[base + 112] = d_hi[0]; - q[base + 113] = d_hi[1]; - col += 1; + let base = col.strict_mul(2); + let mut state = RvvState { + a_lo: load_pair(&q, base), + a_hi: load_pair(&q, base.strict_add(16)), + b_lo: load_pair(&q, base.strict_add(32)), + b_hi: load_pair(&q, base.strict_add(48)), + c_lo: load_pair(&q, base.strict_add(64)), + c_hi: load_pair(&q, base.strict_add(80)), + d_lo: load_pair(&q, base.strict_add(96)), + d_hi: load_pair(&q, base.strict_add(112)), + }; + + p_round(&mut state); + + store_pair(&mut q, base, state.a_lo); + store_pair(&mut q, base.strict_add(16), state.a_hi); + store_pair(&mut q, base.strict_add(32), state.b_lo); + store_pair(&mut q, base.strict_add(48), state.b_hi); + store_pair(&mut q, base.strict_add(64), state.c_lo); + store_pair(&mut q, base.strict_add(80), state.c_hi); + store_pair(&mut q, base.strict_add(96), state.d_lo); + store_pair(&mut q, base.strict_add(112), state.d_hi); + col = col.strict_add(1); } // Final XOR with R, fused with dst store/xor. - let mut i = 0; + let mut i = 0usize; while i < BLOCK_WORDS { - let f0 = q[i] ^ r[i]; - let f1 = q[i + 1] ^ r[i + 1]; - if xor_into { - dst[i] ^= f0; - dst[i + 1] ^= f1; + let result = vxor(load_pair(&q, i), load_pair(&r, i)); + let result = if xor_into { + vxor(load_pair(dst, i), result) } else { - dst[i] = f0; - dst[i + 1] = f1; - } - i += 2; + result + }; + store_pair(dst, i, result); + i = i.strict_add(2); } } +#[inline(always)] +fn load_pair(buf: &[u64; BLOCK_WORDS], idx: usize) -> Pair { + u64x2::from_array([buf[idx], buf[idx.strict_add(1)]]) +} + +#[inline(always)] +fn store_pair(buf: &mut [u64; BLOCK_WORDS], idx: usize, pair: Pair) { + let lanes = pair.to_array(); + buf[idx] = lanes[0]; + buf[idx.strict_add(1)] = lanes[1]; +} + // ─── 4-way P-round ───────────────────────────────────────────────────────── #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn p_round( - a_lo: &mut Pair, - a_hi: &mut Pair, - b_lo: &mut Pair, - b_hi: &mut Pair, - c_lo: &mut Pair, - c_hi: &mut Pair, - d_lo: &mut Pair, - d_hi: &mut Pair, -) { +fn p_round(state: &mut RvvState) { // Column step. - gb(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi); + gb(state); // Diagonalise: rotate b by 1, c by 2, d by 3 across the 4-lane row. - let tb_lo = *b_lo; - let tb_hi = *b_hi; - *b_lo = [tb_lo[1], tb_hi[0]]; - *b_hi = [tb_hi[1], tb_lo[0]]; + let b_lo = state.b_lo; + let b_hi = state.b_hi; + state.b_lo = pair_a1_b0(b_lo, b_hi); + state.b_hi = pair_b1_a0(b_lo, b_hi); - core::mem::swap(c_lo, c_hi); + core::mem::swap(&mut state.c_lo, &mut state.c_hi); - let td_lo = *d_lo; - let td_hi = *d_hi; - *d_lo = [td_hi[1], td_lo[0]]; - *d_hi = [td_lo[1], td_hi[0]]; + let d_lo = state.d_lo; + let d_hi = state.d_hi; + state.d_lo = pair_b1_a0(d_lo, d_hi); + state.d_hi = pair_a1_b0(d_lo, d_hi); // Diagonal step. - gb(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi); + gb(state); // Undo diagonalisation. - let tb_lo = *b_lo; - let tb_hi = *b_hi; - *b_lo = [tb_hi[1], tb_lo[0]]; - *b_hi = [tb_lo[1], tb_hi[0]]; + let b_lo = state.b_lo; + let b_hi = state.b_hi; + state.b_lo = pair_b1_a0(b_lo, b_hi); + state.b_hi = pair_a1_b0(b_lo, b_hi); - core::mem::swap(c_lo, c_hi); + core::mem::swap(&mut state.c_lo, &mut state.c_hi); - let td_lo = *d_lo; - let td_hi = *d_hi; - *d_lo = [td_lo[1], td_hi[0]]; - *d_hi = [td_hi[1], td_lo[0]]; + let d_lo = state.d_lo; + let d_hi = state.d_hi; + state.d_lo = pair_a1_b0(d_lo, d_hi); + state.d_hi = pair_b1_a0(d_lo, d_hi); +} + +#[inline(always)] +fn pair_a1_b0(a: Pair, b: Pair) -> Pair { + core::simd::simd_swizzle!(a, b, [1, 2]) +} + +#[inline(always)] +fn pair_b1_a0(a: Pair, b: Pair) -> Pair { + core::simd::simd_swizzle!(a, b, [3, 0]) } // ─── 4-way BlaMka G ──────────────────────────────────────────────────────── #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn gb( - a_lo: &mut Pair, - a_hi: &mut Pair, - b_lo: &mut Pair, - b_hi: &mut Pair, - c_lo: &mut Pair, - c_hi: &mut Pair, - d_lo: &mut Pair, - d_hi: &mut Pair, -) { +fn gb(state: &mut RvvState) { // Step 1: a = a + b + 2·lsb(a)·lsb(b) - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = vadd(vadd(*a_lo, *b_lo), p_lo); - *a_hi = vadd(vadd(*a_hi, *b_hi), p_hi); - *d_lo = ror(vxor(*d_lo, *a_lo), 32); - *d_hi = ror(vxor(*d_hi, *a_hi), 32); + let p_lo = bla_mul(state.a_lo, state.b_lo); + let p_hi = bla_mul(state.a_hi, state.b_hi); + state.a_lo = bla_add(state.a_lo, state.b_lo, p_lo); + state.a_hi = bla_add(state.a_hi, state.b_hi, p_hi); + state.d_lo = ror::<32>(vxor(state.d_lo, state.a_lo)); + state.d_hi = ror::<32>(vxor(state.d_hi, state.a_hi)); // Step 2: c = c + d + 2·lsb(c)·lsb(d) - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = vadd(vadd(*c_lo, *d_lo), p_lo); - *c_hi = vadd(vadd(*c_hi, *d_hi), p_hi); - *b_lo = ror(vxor(*b_lo, *c_lo), 24); - *b_hi = ror(vxor(*b_hi, *c_hi), 24); + let p_lo = bla_mul(state.c_lo, state.d_lo); + let p_hi = bla_mul(state.c_hi, state.d_hi); + state.c_lo = bla_add(state.c_lo, state.d_lo, p_lo); + state.c_hi = bla_add(state.c_hi, state.d_hi, p_hi); + state.b_lo = ror::<24>(vxor(state.b_lo, state.c_lo)); + state.b_hi = ror::<24>(vxor(state.b_hi, state.c_hi)); // Step 3: a = a + b + 2·lsb(a)·lsb(b) - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = vadd(vadd(*a_lo, *b_lo), p_lo); - *a_hi = vadd(vadd(*a_hi, *b_hi), p_hi); - *d_lo = ror(vxor(*d_lo, *a_lo), 16); - *d_hi = ror(vxor(*d_hi, *a_hi), 16); + let p_lo = bla_mul(state.a_lo, state.b_lo); + let p_hi = bla_mul(state.a_hi, state.b_hi); + state.a_lo = bla_add(state.a_lo, state.b_lo, p_lo); + state.a_hi = bla_add(state.a_hi, state.b_hi, p_hi); + state.d_lo = ror::<16>(vxor(state.d_lo, state.a_lo)); + state.d_hi = ror::<16>(vxor(state.d_hi, state.a_hi)); // Step 4: c = c + d + 2·lsb(c)·lsb(d) - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = vadd(vadd(*c_lo, *d_lo), p_lo); - *c_hi = vadd(vadd(*c_hi, *d_hi), p_hi); - *b_lo = ror(vxor(*b_lo, *c_lo), 63); - *b_hi = ror(vxor(*b_hi, *c_hi), 63); + let p_lo = bla_mul(state.c_lo, state.d_lo); + let p_hi = bla_mul(state.c_hi, state.d_hi); + state.c_lo = bla_add(state.c_lo, state.d_lo, p_lo); + state.c_hi = bla_add(state.c_hi, state.d_hi, p_hi); + state.b_lo = ror::<63>(vxor(state.b_lo, state.c_lo)); + state.b_hi = ror::<63>(vxor(state.b_hi, state.c_hi)); } -// ─── Micro-ops (compiler-vectorised at VL=2/SEW=64 under -C target-feature=+v) ─ +// ─── Micro-ops ───────────────────────────────────────────────────────────── #[inline(always)] -fn vadd(a: Pair, b: Pair) -> Pair { - [a[0].wrapping_add(b[0]), a[1].wrapping_add(b[1])] +fn vxor(a: Pair, b: Pair) -> Pair { + core::ops::BitXor::bitxor(a, b) } #[inline(always)] -fn vxor(a: Pair, b: Pair) -> Pair { - [a[0] ^ b[0], a[1] ^ b[1]] +fn ror(value: Pair) -> Pair { + const { assert!(N > 0 && N < 64) } + let right = core::ops::Shr::shr(value, u64x2::splat(u64::from(N))); + let left = core::ops::Shl::shl(value, u64x2::splat(u64::from(64u32.strict_sub(N)))); + core::ops::BitOr::bitor(right, left) } #[inline(always)] -fn ror(x: Pair, n: u32) -> Pair { - [x[0].rotate_right(n), x[1].rotate_right(n)] +fn bla_add(a: Pair, b: Pair, product: Pair) -> Pair { + simd_wrapping_add(simd_wrapping_add(a, b), product) } /// `2 · lsb(a) · lsb(b)` lane-wise. Masked u32 multiply fits in u64 so /// the result is exact without a 128-bit-wide product. #[inline(always)] fn bla_mul(a: Pair, b: Pair) -> Pair { - const MASK: u64 = 0xffff_ffff; - [ - (a[0] & MASK).wrapping_mul(b[0] & MASK).wrapping_shl(1), - (a[1] & MASK).wrapping_mul(b[1] & MASK).wrapping_shl(1), - ] + let mask = u64x2::splat(0xffff_ffff); + let product = simd_wrapping_mul(a & mask, b & mask); + core::ops::Shl::shl(product, u64x2::splat(1)) +} + +#[inline(always)] +fn simd_wrapping_add(a: Pair, b: Pair) -> Pair { + core::ops::Add::add(a, b) +} + +#[inline(always)] +fn simd_wrapping_mul(a: Pair, b: Pair) -> Pair { + core::ops::Mul::mul(a, b) } diff --git a/src/auth/argon2/s390x.rs b/src/auth/argon2/s390x.rs index 310023cb..7749bb98 100644 --- a/src/auth/argon2/s390x.rs +++ b/src/auth/argon2/s390x.rs @@ -27,16 +27,28 @@ //! `s390x::VECTOR`. #![cfg(target_arch = "s390x")] -#![allow(unsafe_code)] -#![allow(clippy::cast_possible_truncation)] use core::simd::i64x2; use super::BLOCK_WORDS; +struct VectorState { + a_lo: i64x2, + a_hi: i64x2, + b_lo: i64x2, + b_hi: i64x2, + c_lo: i64x2, + c_hi: i64x2, + d_lo: i64x2, + d_hi: i64x2, +} + // ─── Inline-asm primitives (z13+ vector facility) ────────────────────────── #[target_feature(enable = "vector")] +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. unsafe fn vag(a: i64x2, b: i64x2) -> i64x2 { let out: i64x2; // SAFETY: z13+ vector facility via target_feature. @@ -53,6 +65,9 @@ unsafe fn vag(a: i64x2, b: i64x2) -> i64x2 { } #[target_feature(enable = "vector")] +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. unsafe fn vx(a: i64x2, b: i64x2) -> i64x2 { let out: i64x2; // SAFETY: z13+ vector facility via target_feature. @@ -69,6 +84,10 @@ unsafe fn vx(a: i64x2, b: i64x2) -> i64x2 { } /// `verllg` ROL by 32 = ROR 32. +/// +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. #[target_feature(enable = "vector")] unsafe fn verllg_32(x: i64x2) -> i64x2 { let out: i64x2; @@ -85,6 +104,10 @@ unsafe fn verllg_32(x: i64x2) -> i64x2 { } /// `verllg` ROL by 40 = ROR 24. +/// +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. #[target_feature(enable = "vector")] unsafe fn verllg_40(x: i64x2) -> i64x2 { let out: i64x2; @@ -101,6 +124,10 @@ unsafe fn verllg_40(x: i64x2) -> i64x2 { } /// `verllg` ROL by 48 = ROR 16. +/// +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. #[target_feature(enable = "vector")] unsafe fn verllg_48(x: i64x2) -> i64x2 { let out: i64x2; @@ -117,6 +144,10 @@ unsafe fn verllg_48(x: i64x2) -> i64x2 { } /// `verllg` ROL by 1 = ROR 63. +/// +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. #[target_feature(enable = "vector")] unsafe fn verllg_1(x: i64x2) -> i64x2 { let out: i64x2; @@ -143,15 +174,22 @@ fn pair_b1_a0(a: i64x2, b: i64x2) -> i64x2 { } #[inline(always)] +/// # Safety +/// +/// `p` must remain valid to read two initialized `u64` values from one +/// allocation. unsafe fn vload_pair(p: *const u64) -> i64x2 { // SAFETY: caller ensures p is valid for 2 × u64. - unsafe { core::ptr::read_unaligned(p as *const i64x2) } + unsafe { core::ptr::read_unaligned(p.cast()) } } #[inline(always)] +/// # Safety +/// +/// `p` must remain valid to write two `u64` values into one allocation. unsafe fn vstore_pair(p: *mut u64, v: i64x2) { // SAFETY: caller ensures p is valid for 2 × u64. - unsafe { core::ptr::write_unaligned(p as *mut i64x2, v) } + unsafe { core::ptr::write_unaligned(p.cast(), v) } } /// `2 · lsb(a) · lsb(b)` lane-wise — scalar fallback (z13 has no native @@ -162,108 +200,94 @@ fn bla_mul(a: i64x2, b: i64x2) -> i64x2 { let aa = a.to_array(); let bb = b.to_array(); const MASK: u64 = 0xffff_ffff; - let r0 = ((aa[0] as u64) & MASK) - .wrapping_mul((bb[0] as u64) & MASK) + let r0 = (aa[0].cast_unsigned() & MASK) + .wrapping_mul(bb[0].cast_unsigned() & MASK) .wrapping_shl(1); - let r1 = ((aa[1] as u64) & MASK) - .wrapping_mul((bb[1] as u64) & MASK) + let r1 = (aa[1].cast_unsigned() & MASK) + .wrapping_mul(bb[1].cast_unsigned() & MASK) .wrapping_shl(1); - i64x2::from_array([r0 as i64, r1 as i64]) + i64x2::from_array([r0.cast_signed(), r1.cast_signed()]) } // ─── 4-way P-round ───────────────────────────────────────────────────────── -#[allow(clippy::too_many_arguments)] +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. #[target_feature(enable = "vector")] -unsafe fn p_round( - a_lo: &mut i64x2, - a_hi: &mut i64x2, - b_lo: &mut i64x2, - b_hi: &mut i64x2, - c_lo: &mut i64x2, - c_hi: &mut i64x2, - d_lo: &mut i64x2, - d_hi: &mut i64x2, -) { +unsafe fn p_round(state: &mut VectorState) { // SAFETY: vector facility inherited. unsafe { - gb(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi); + gb(state); - let tb_lo = *b_lo; - let tb_hi = *b_hi; - *b_lo = pair_a1_b0(tb_lo, tb_hi); - *b_hi = pair_b1_a0(tb_lo, tb_hi); + let b_lo = state.b_lo; + let b_hi = state.b_hi; + state.b_lo = pair_a1_b0(b_lo, b_hi); + state.b_hi = pair_b1_a0(b_lo, b_hi); - core::mem::swap(c_lo, c_hi); + core::mem::swap(&mut state.c_lo, &mut state.c_hi); - let td_lo = *d_lo; - let td_hi = *d_hi; - *d_lo = pair_b1_a0(td_lo, td_hi); - *d_hi = pair_a1_b0(td_lo, td_hi); + let d_lo = state.d_lo; + let d_hi = state.d_hi; + state.d_lo = pair_b1_a0(d_lo, d_hi); + state.d_hi = pair_a1_b0(d_lo, d_hi); - gb(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi); + gb(state); - let tb_lo = *b_lo; - let tb_hi = *b_hi; - *b_lo = pair_b1_a0(tb_lo, tb_hi); - *b_hi = pair_a1_b0(tb_lo, tb_hi); + let b_lo = state.b_lo; + let b_hi = state.b_hi; + state.b_lo = pair_b1_a0(b_lo, b_hi); + state.b_hi = pair_a1_b0(b_lo, b_hi); - core::mem::swap(c_lo, c_hi); + core::mem::swap(&mut state.c_lo, &mut state.c_hi); - let td_lo = *d_lo; - let td_hi = *d_hi; - *d_lo = pair_a1_b0(td_lo, td_hi); - *d_hi = pair_b1_a0(td_lo, td_hi); + let d_lo = state.d_lo; + let d_hi = state.d_hi; + state.d_lo = pair_a1_b0(d_lo, d_hi); + state.d_hi = pair_b1_a0(d_lo, d_hi); } } // ─── 4-way BlaMka G ──────────────────────────────────────────────────────── -#[allow(clippy::too_many_arguments)] +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. #[target_feature(enable = "vector")] -unsafe fn gb( - a_lo: &mut i64x2, - a_hi: &mut i64x2, - b_lo: &mut i64x2, - b_hi: &mut i64x2, - c_lo: &mut i64x2, - c_hi: &mut i64x2, - d_lo: &mut i64x2, - d_hi: &mut i64x2, -) { +unsafe fn gb(state: &mut VectorState) { // SAFETY: vector facility inherited. unsafe { // Step 1 - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = vag(vag(*a_lo, *b_lo), p_lo); - *a_hi = vag(vag(*a_hi, *b_hi), p_hi); - *d_lo = verllg_32(vx(*d_lo, *a_lo)); - *d_hi = verllg_32(vx(*d_hi, *a_hi)); + let p_lo = bla_mul(state.a_lo, state.b_lo); + let p_hi = bla_mul(state.a_hi, state.b_hi); + state.a_lo = vag(vag(state.a_lo, state.b_lo), p_lo); + state.a_hi = vag(vag(state.a_hi, state.b_hi), p_hi); + state.d_lo = verllg_32(vx(state.d_lo, state.a_lo)); + state.d_hi = verllg_32(vx(state.d_hi, state.a_hi)); // Step 2 - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = vag(vag(*c_lo, *d_lo), p_lo); - *c_hi = vag(vag(*c_hi, *d_hi), p_hi); - *b_lo = verllg_40(vx(*b_lo, *c_lo)); - *b_hi = verllg_40(vx(*b_hi, *c_hi)); + let p_lo = bla_mul(state.c_lo, state.d_lo); + let p_hi = bla_mul(state.c_hi, state.d_hi); + state.c_lo = vag(vag(state.c_lo, state.d_lo), p_lo); + state.c_hi = vag(vag(state.c_hi, state.d_hi), p_hi); + state.b_lo = verllg_40(vx(state.b_lo, state.c_lo)); + state.b_hi = verllg_40(vx(state.b_hi, state.c_hi)); // Step 3 - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = vag(vag(*a_lo, *b_lo), p_lo); - *a_hi = vag(vag(*a_hi, *b_hi), p_hi); - *d_lo = verllg_48(vx(*d_lo, *a_lo)); - *d_hi = verllg_48(vx(*d_hi, *a_hi)); + let p_lo = bla_mul(state.a_lo, state.b_lo); + let p_hi = bla_mul(state.a_hi, state.b_hi); + state.a_lo = vag(vag(state.a_lo, state.b_lo), p_lo); + state.a_hi = vag(vag(state.a_hi, state.b_hi), p_hi); + state.d_lo = verllg_48(vx(state.d_lo, state.a_lo)); + state.d_hi = verllg_48(vx(state.d_hi, state.a_hi)); // Step 4 - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = vag(vag(*c_lo, *d_lo), p_lo); - *c_hi = vag(vag(*c_hi, *d_hi), p_hi); - *b_lo = verllg_1(vx(*b_lo, *c_lo)); - *b_hi = verllg_1(vx(*b_hi, *c_hi)); + let p_lo = bla_mul(state.c_lo, state.d_lo); + let p_hi = bla_mul(state.c_hi, state.d_hi); + state.c_lo = vag(vag(state.c_lo, state.d_lo), p_lo); + state.c_hi = vag(vag(state.c_hi, state.d_hi), p_hi); + state.b_lo = verllg_1(vx(state.b_lo, state.c_lo)); + state.b_hi = verllg_1(vx(state.b_hi, state.c_hi)); } } @@ -289,74 +313,74 @@ pub(super) unsafe fn compress_vector( unsafe { let mut r = [0u64; BLOCK_WORDS]; let mut q = [0u64; BLOCK_WORDS]; - let mut i = 0; + let mut i = 0usize; while i < BLOCK_WORDS { let xv = vload_pair(x.as_ptr().add(i)); let yv = vload_pair(y.as_ptr().add(i)); let rv = vx(xv, yv); vstore_pair(r.as_mut_ptr().add(i), rv); vstore_pair(q.as_mut_ptr().add(i), rv); - i += 2; + i = i.strict_add(2); } // Row pass. let mut row = 0usize; while row < 8 { - let base = row * 16; - let mut a_lo = vload_pair(q.as_ptr().add(base)); - let mut a_hi = vload_pair(q.as_ptr().add(base + 2)); - let mut b_lo = vload_pair(q.as_ptr().add(base + 4)); - let mut b_hi = vload_pair(q.as_ptr().add(base + 6)); - let mut c_lo = vload_pair(q.as_ptr().add(base + 8)); - let mut c_hi = vload_pair(q.as_ptr().add(base + 10)); - let mut d_lo = vload_pair(q.as_ptr().add(base + 12)); - let mut d_hi = vload_pair(q.as_ptr().add(base + 14)); - - p_round( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - - vstore_pair(q.as_mut_ptr().add(base), a_lo); - vstore_pair(q.as_mut_ptr().add(base + 2), a_hi); - vstore_pair(q.as_mut_ptr().add(base + 4), b_lo); - vstore_pair(q.as_mut_ptr().add(base + 6), b_hi); - vstore_pair(q.as_mut_ptr().add(base + 8), c_lo); - vstore_pair(q.as_mut_ptr().add(base + 10), c_hi); - vstore_pair(q.as_mut_ptr().add(base + 12), d_lo); - vstore_pair(q.as_mut_ptr().add(base + 14), d_hi); - row += 1; + let base = row.strict_mul(16); + let mut state = VectorState { + a_lo: vload_pair(q.as_ptr().add(base)), + a_hi: vload_pair(q.as_ptr().add(base.strict_add(2))), + b_lo: vload_pair(q.as_ptr().add(base.strict_add(4))), + b_hi: vload_pair(q.as_ptr().add(base.strict_add(6))), + c_lo: vload_pair(q.as_ptr().add(base.strict_add(8))), + c_hi: vload_pair(q.as_ptr().add(base.strict_add(10))), + d_lo: vload_pair(q.as_ptr().add(base.strict_add(12))), + d_hi: vload_pair(q.as_ptr().add(base.strict_add(14))), + }; + + p_round(&mut state); + + vstore_pair(q.as_mut_ptr().add(base), state.a_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(2)), state.a_hi); + vstore_pair(q.as_mut_ptr().add(base.strict_add(4)), state.b_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(6)), state.b_hi); + vstore_pair(q.as_mut_ptr().add(base.strict_add(8)), state.c_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(10)), state.c_hi); + vstore_pair(q.as_mut_ptr().add(base.strict_add(12)), state.d_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(14)), state.d_hi); + row = row.strict_add(1); } // Column pass. let mut col = 0usize; while col < 8 { - let base = col * 2; - let mut a_lo = vload_pair(q.as_ptr().add(base)); - let mut a_hi = vload_pair(q.as_ptr().add(base + 16)); - let mut b_lo = vload_pair(q.as_ptr().add(base + 32)); - let mut b_hi = vload_pair(q.as_ptr().add(base + 48)); - let mut c_lo = vload_pair(q.as_ptr().add(base + 64)); - let mut c_hi = vload_pair(q.as_ptr().add(base + 80)); - let mut d_lo = vload_pair(q.as_ptr().add(base + 96)); - let mut d_hi = vload_pair(q.as_ptr().add(base + 112)); - - p_round( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - - vstore_pair(q.as_mut_ptr().add(base), a_lo); - vstore_pair(q.as_mut_ptr().add(base + 16), a_hi); - vstore_pair(q.as_mut_ptr().add(base + 32), b_lo); - vstore_pair(q.as_mut_ptr().add(base + 48), b_hi); - vstore_pair(q.as_mut_ptr().add(base + 64), c_lo); - vstore_pair(q.as_mut_ptr().add(base + 80), c_hi); - vstore_pair(q.as_mut_ptr().add(base + 96), d_lo); - vstore_pair(q.as_mut_ptr().add(base + 112), d_hi); - col += 1; + let base = col.strict_mul(2); + let mut state = VectorState { + a_lo: vload_pair(q.as_ptr().add(base)), + a_hi: vload_pair(q.as_ptr().add(base.strict_add(16))), + b_lo: vload_pair(q.as_ptr().add(base.strict_add(32))), + b_hi: vload_pair(q.as_ptr().add(base.strict_add(48))), + c_lo: vload_pair(q.as_ptr().add(base.strict_add(64))), + c_hi: vload_pair(q.as_ptr().add(base.strict_add(80))), + d_lo: vload_pair(q.as_ptr().add(base.strict_add(96))), + d_hi: vload_pair(q.as_ptr().add(base.strict_add(112))), + }; + + p_round(&mut state); + + vstore_pair(q.as_mut_ptr().add(base), state.a_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(16)), state.a_hi); + vstore_pair(q.as_mut_ptr().add(base.strict_add(32)), state.b_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(48)), state.b_hi); + vstore_pair(q.as_mut_ptr().add(base.strict_add(64)), state.c_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(80)), state.c_hi); + vstore_pair(q.as_mut_ptr().add(base.strict_add(96)), state.d_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(112)), state.d_hi); + col = col.strict_add(1); } // Final XOR with R, fused with dst store/xor. - let mut i = 0; + let mut i = 0usize; while i < BLOCK_WORDS { let qv = vload_pair(q.as_ptr().add(i)); let rv = vload_pair(r.as_ptr().add(i)); @@ -367,7 +391,7 @@ pub(super) unsafe fn compress_vector( } else { vstore_pair(dst.as_mut_ptr().add(i), f); } - i += 2; + i = i.strict_add(2); } } } diff --git a/src/auth/argon2/wasm.rs b/src/auth/argon2/wasm.rs index 49c314ea..5c61e2c3 100644 --- a/src/auth/argon2/wasm.rs +++ b/src/auth/argon2/wasm.rs @@ -36,15 +36,32 @@ //! - ROR 63 ≡ ROL 1: shift-right + shift-left + OR. #![cfg(target_arch = "wasm32")] -#![allow(clippy::cast_possible_truncation)] use core::arch::wasm32::{ - i8x16_shuffle, i64x2_add, i64x2_mul, i64x2_shuffle, u64x2_shl, u64x2_shr, u64x2_splat, v128, v128_and, v128_load, - v128_or, v128_store, v128_xor, + i8x16_shuffle, i64x2_add, i64x2_mul, i64x2_shuffle, u64x2, u64x2_extract_lane, u64x2_shl, u64x2_shr, u64x2_splat, + v128, v128_and, v128_or, v128_xor, }; use super::BLOCK_WORDS; +struct RoundState { + a: [v128; 2], + b: [v128; 2], + c: [v128; 2], + d: [v128; 2], +} + +#[inline(always)] +fn load_pair(words: &[u64; N], offset: usize) -> v128 { + u64x2(words[offset], words[offset.strict_add(1)]) +} + +#[inline(always)] +fn store_pair(words: &mut [u64; N], offset: usize, value: v128) { + words[offset] = u64x2_extract_lane::<0>(value); + words[offset.strict_add(1)] = u64x2_extract_lane::<1>(value); +} + /// WebAssembly SIMD128 BlaMka compression kernel. /// /// # Safety @@ -60,190 +77,150 @@ pub(super) unsafe fn compress_simd128( y: &[u64; BLOCK_WORDS], xor_into: bool, ) { - // SAFETY: simd128 is enabled by this function's `#[target_feature]` - // attribute, so all `v128`/`u64x2_*` ops below are valid to call. - unsafe { - // R = X XOR Y, materialised to scratch for re-reads during the - // row + column passes plus the final XOR. - let mut r = [0u64; BLOCK_WORDS]; - let mut q = [0u64; BLOCK_WORDS]; - let mut i = 0; - while i < BLOCK_WORDS { - let xv = v128_load(x.as_ptr().add(i).cast()); - let yv = v128_load(y.as_ptr().add(i).cast()); - let rv = v128_xor(xv, yv); - v128_store(r.as_mut_ptr().add(i).cast(), rv); - v128_store(q.as_mut_ptr().add(i).cast(), rv); - i += 2; - } - - // Row pass: 8 P-rounds on contiguous 16-u64 chunks of q[]. - let mut row = 0usize; - while row < 8 { - let base = row * 16; - let mut a_lo = v128_load(q.as_ptr().add(base).cast()); - let mut a_hi = v128_load(q.as_ptr().add(base + 2).cast()); - let mut b_lo = v128_load(q.as_ptr().add(base + 4).cast()); - let mut b_hi = v128_load(q.as_ptr().add(base + 6).cast()); - let mut c_lo = v128_load(q.as_ptr().add(base + 8).cast()); - let mut c_hi = v128_load(q.as_ptr().add(base + 10).cast()); - let mut d_lo = v128_load(q.as_ptr().add(base + 12).cast()); - let mut d_hi = v128_load(q.as_ptr().add(base + 14).cast()); - - p_round( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - - v128_store(q.as_mut_ptr().add(base).cast(), a_lo); - v128_store(q.as_mut_ptr().add(base + 2).cast(), a_hi); - v128_store(q.as_mut_ptr().add(base + 4).cast(), b_lo); - v128_store(q.as_mut_ptr().add(base + 6).cast(), b_hi); - v128_store(q.as_mut_ptr().add(base + 8).cast(), c_lo); - v128_store(q.as_mut_ptr().add(base + 10).cast(), c_hi); - v128_store(q.as_mut_ptr().add(base + 12).cast(), d_lo); - v128_store(q.as_mut_ptr().add(base + 14).cast(), d_hi); - row += 1; - } - - // Column pass: 8 P-rounds on stride-16 u64 sequences. Each lane of - // 2 u64 is loaded directly from the natural row-major positions — - // see RFC 9106 §3.6 column-step indexing. - let mut col = 0usize; - while col < 8 { - let base = col * 2; - let mut a_lo = v128_load(q.as_ptr().add(base).cast()); - let mut a_hi = v128_load(q.as_ptr().add(base + 16).cast()); - let mut b_lo = v128_load(q.as_ptr().add(base + 32).cast()); - let mut b_hi = v128_load(q.as_ptr().add(base + 48).cast()); - let mut c_lo = v128_load(q.as_ptr().add(base + 64).cast()); - let mut c_hi = v128_load(q.as_ptr().add(base + 80).cast()); - let mut d_lo = v128_load(q.as_ptr().add(base + 96).cast()); - let mut d_hi = v128_load(q.as_ptr().add(base + 112).cast()); - - p_round( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - - v128_store(q.as_mut_ptr().add(base).cast(), a_lo); - v128_store(q.as_mut_ptr().add(base + 16).cast(), a_hi); - v128_store(q.as_mut_ptr().add(base + 32).cast(), b_lo); - v128_store(q.as_mut_ptr().add(base + 48).cast(), b_hi); - v128_store(q.as_mut_ptr().add(base + 64).cast(), c_lo); - v128_store(q.as_mut_ptr().add(base + 80).cast(), c_hi); - v128_store(q.as_mut_ptr().add(base + 96).cast(), d_lo); - v128_store(q.as_mut_ptr().add(base + 112).cast(), d_hi); - col += 1; - } - - // Final XOR with R, fused with dst store/xor. - let mut i = 0; - while i < BLOCK_WORDS { - let qv = v128_load(q.as_ptr().add(i).cast()); - let rv = v128_load(r.as_ptr().add(i).cast()); - let f = v128_xor(qv, rv); - if xor_into { - let cur = v128_load(dst.as_ptr().add(i).cast()); - v128_store(dst.as_mut_ptr().add(i).cast(), v128_xor(cur, f)); - } else { - v128_store(dst.as_mut_ptr().add(i).cast(), f); - } - i += 2; - } + // R = X XOR Y, materialised to scratch for re-reads during the + // row + column passes plus the final XOR. + let mut r = [0u64; BLOCK_WORDS]; + let mut q = [0u64; BLOCK_WORDS]; + let mut i = 0usize; + while i < BLOCK_WORDS { + let rv = v128_xor(load_pair(x, i), load_pair(y, i)); + store_pair(&mut r, i, rv); + store_pair(&mut q, i, rv); + i = i.strict_add(2); + } + + // Row pass: 8 P-rounds on contiguous 16-u64 chunks of q[]. + let mut row = 0usize; + while row < 8 { + let base = row.strict_mul(16); + let mut state = RoundState { + a: [load_pair(&q, base), load_pair(&q, base.strict_add(2))], + b: [load_pair(&q, base.strict_add(4)), load_pair(&q, base.strict_add(6))], + c: [load_pair(&q, base.strict_add(8)), load_pair(&q, base.strict_add(10))], + d: [load_pair(&q, base.strict_add(12)), load_pair(&q, base.strict_add(14))], + }; + + p_round(&mut state); + + store_pair(&mut q, base, state.a[0]); + store_pair(&mut q, base.strict_add(2), state.a[1]); + store_pair(&mut q, base.strict_add(4), state.b[0]); + store_pair(&mut q, base.strict_add(6), state.b[1]); + store_pair(&mut q, base.strict_add(8), state.c[0]); + store_pair(&mut q, base.strict_add(10), state.c[1]); + store_pair(&mut q, base.strict_add(12), state.d[0]); + store_pair(&mut q, base.strict_add(14), state.d[1]); + row = row.strict_add(1); + } + + // Column pass: 8 P-rounds on stride-16 u64 sequences. Each lane of + // 2 u64 is loaded directly from the natural row-major positions — + // see RFC 9106 §3.6 column-step indexing. + let mut col = 0usize; + while col < 8 { + let base = col.strict_mul(2); + let mut state = RoundState { + a: [load_pair(&q, base), load_pair(&q, base.strict_add(16))], + b: [load_pair(&q, base.strict_add(32)), load_pair(&q, base.strict_add(48))], + c: [load_pair(&q, base.strict_add(64)), load_pair(&q, base.strict_add(80))], + d: [load_pair(&q, base.strict_add(96)), load_pair(&q, base.strict_add(112))], + }; + + p_round(&mut state); + + store_pair(&mut q, base, state.a[0]); + store_pair(&mut q, base.strict_add(16), state.a[1]); + store_pair(&mut q, base.strict_add(32), state.b[0]); + store_pair(&mut q, base.strict_add(48), state.b[1]); + store_pair(&mut q, base.strict_add(64), state.c[0]); + store_pair(&mut q, base.strict_add(80), state.c[1]); + store_pair(&mut q, base.strict_add(96), state.d[0]); + store_pair(&mut q, base.strict_add(112), state.d[1]); + col = col.strict_add(1); + } + + // Final XOR with R, fused with dst store/xor. + let mut i = 0usize; + while i < BLOCK_WORDS { + let f = v128_xor(load_pair(&q, i), load_pair(&r, i)); + let output = if xor_into { v128_xor(load_pair(dst, i), f) } else { f }; + store_pair(dst, i, output); + i = i.strict_add(2); } } // ─── 4-way P-round ───────────────────────────────────────────────────────── #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn p_round( - a_lo: &mut v128, - a_hi: &mut v128, - b_lo: &mut v128, - b_hi: &mut v128, - c_lo: &mut v128, - c_hi: &mut v128, - d_lo: &mut v128, - d_hi: &mut v128, -) { +fn p_round(state: &mut RoundState) { // Column step. - gb(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi); + gb(state); // Diagonalise: rotate b by 1, c by 2, d by 3 across the 4-lane row. - let tb_lo = *b_lo; - let tb_hi = *b_hi; - *b_lo = i64x2_shuffle::<1, 2>(tb_lo, tb_hi); - *b_hi = i64x2_shuffle::<1, 2>(tb_hi, tb_lo); + let tb_lo = state.b[0]; + let tb_hi = state.b[1]; + state.b[0] = i64x2_shuffle::<1, 2>(tb_lo, tb_hi); + state.b[1] = i64x2_shuffle::<1, 2>(tb_hi, tb_lo); - core::mem::swap(c_lo, c_hi); + state.c.swap(0, 1); - let td_lo = *d_lo; - let td_hi = *d_hi; - *d_lo = i64x2_shuffle::<1, 2>(td_hi, td_lo); - *d_hi = i64x2_shuffle::<1, 2>(td_lo, td_hi); + let td_lo = state.d[0]; + let td_hi = state.d[1]; + state.d[0] = i64x2_shuffle::<1, 2>(td_hi, td_lo); + state.d[1] = i64x2_shuffle::<1, 2>(td_lo, td_hi); // Diagonal step. - gb(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi); + gb(state); // Undo diagonalisation. - let tb_lo = *b_lo; - let tb_hi = *b_hi; - *b_lo = i64x2_shuffle::<1, 2>(tb_hi, tb_lo); - *b_hi = i64x2_shuffle::<1, 2>(tb_lo, tb_hi); + let tb_lo = state.b[0]; + let tb_hi = state.b[1]; + state.b[0] = i64x2_shuffle::<1, 2>(tb_hi, tb_lo); + state.b[1] = i64x2_shuffle::<1, 2>(tb_lo, tb_hi); - core::mem::swap(c_lo, c_hi); + state.c.swap(0, 1); - let td_lo = *d_lo; - let td_hi = *d_hi; - *d_lo = i64x2_shuffle::<1, 2>(td_lo, td_hi); - *d_hi = i64x2_shuffle::<1, 2>(td_hi, td_lo); + let td_lo = state.d[0]; + let td_hi = state.d[1]; + state.d[0] = i64x2_shuffle::<1, 2>(td_lo, td_hi); + state.d[1] = i64x2_shuffle::<1, 2>(td_hi, td_lo); } // ─── 4-way BlaMka G ──────────────────────────────────────────────────────── #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn gb( - a_lo: &mut v128, - a_hi: &mut v128, - b_lo: &mut v128, - b_hi: &mut v128, - c_lo: &mut v128, - c_hi: &mut v128, - d_lo: &mut v128, - d_hi: &mut v128, -) { +fn gb(state: &mut RoundState) { // Step 1: a = a + b + 2·lsb(a)·lsb(b) - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = i64x2_add(i64x2_add(*a_lo, *b_lo), p_lo); - *a_hi = i64x2_add(i64x2_add(*a_hi, *b_hi), p_hi); - *d_lo = ror32(v128_xor(*d_lo, *a_lo)); - *d_hi = ror32(v128_xor(*d_hi, *a_hi)); + let p_lo = bla_mul(state.a[0], state.b[0]); + let p_hi = bla_mul(state.a[1], state.b[1]); + state.a[0] = i64x2_add(i64x2_add(state.a[0], state.b[0]), p_lo); + state.a[1] = i64x2_add(i64x2_add(state.a[1], state.b[1]), p_hi); + state.d[0] = ror32(v128_xor(state.d[0], state.a[0])); + state.d[1] = ror32(v128_xor(state.d[1], state.a[1])); // Step 2: c = c + d + 2·lsb(c)·lsb(d) - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = i64x2_add(i64x2_add(*c_lo, *d_lo), p_lo); - *c_hi = i64x2_add(i64x2_add(*c_hi, *d_hi), p_hi); - *b_lo = ror24(v128_xor(*b_lo, *c_lo)); - *b_hi = ror24(v128_xor(*b_hi, *c_hi)); + let p_lo = bla_mul(state.c[0], state.d[0]); + let p_hi = bla_mul(state.c[1], state.d[1]); + state.c[0] = i64x2_add(i64x2_add(state.c[0], state.d[0]), p_lo); + state.c[1] = i64x2_add(i64x2_add(state.c[1], state.d[1]), p_hi); + state.b[0] = ror24(v128_xor(state.b[0], state.c[0])); + state.b[1] = ror24(v128_xor(state.b[1], state.c[1])); // Step 3: a = a + b + 2·lsb(a)·lsb(b) - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = i64x2_add(i64x2_add(*a_lo, *b_lo), p_lo); - *a_hi = i64x2_add(i64x2_add(*a_hi, *b_hi), p_hi); - *d_lo = ror16(v128_xor(*d_lo, *a_lo)); - *d_hi = ror16(v128_xor(*d_hi, *a_hi)); + let p_lo = bla_mul(state.a[0], state.b[0]); + let p_hi = bla_mul(state.a[1], state.b[1]); + state.a[0] = i64x2_add(i64x2_add(state.a[0], state.b[0]), p_lo); + state.a[1] = i64x2_add(i64x2_add(state.a[1], state.b[1]), p_hi); + state.d[0] = ror16(v128_xor(state.d[0], state.a[0])); + state.d[1] = ror16(v128_xor(state.d[1], state.a[1])); // Step 4: c = c + d + 2·lsb(c)·lsb(d) - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = i64x2_add(i64x2_add(*c_lo, *d_lo), p_lo); - *c_hi = i64x2_add(i64x2_add(*c_hi, *d_hi), p_hi); - *b_lo = ror63(v128_xor(*b_lo, *c_lo)); - *b_hi = ror63(v128_xor(*b_hi, *c_hi)); + let p_lo = bla_mul(state.c[0], state.d[0]); + let p_hi = bla_mul(state.c[1], state.d[1]); + state.c[0] = i64x2_add(i64x2_add(state.c[0], state.d[0]), p_lo); + state.c[1] = i64x2_add(i64x2_add(state.c[1], state.d[1]), p_hi); + state.b[0] = ror63(v128_xor(state.b[0], state.c[0])); + state.b[1] = ror63(v128_xor(state.b[1], state.c[1])); } // ─── Micro-ops ───────────────────────────────────────────────────────────── diff --git a/src/auth/argon2/x86_64.rs b/src/auth/argon2/x86_64.rs index 10cb9b9f..f24180bd 100644 --- a/src/auth/argon2/x86_64.rs +++ b/src/auth/argon2/x86_64.rs @@ -36,7 +36,6 @@ //! left shift by 1 supplies the `2 · …` factor. #![cfg(target_arch = "x86_64")] -#![allow(clippy::cast_possible_truncation)] use core::arch::x86_64::{ __m256i, __m512i, _mm_loadu_si128, _mm_storeu_si128, _mm256_add_epi64, _mm256_castsi128_si256, @@ -100,25 +99,25 @@ pub(super) unsafe fn compress_avx2( let rv = _mm256_xor_si256(xv, yv); _mm256_storeu_si256(r.as_mut_ptr().add(i).cast(), rv); _mm256_storeu_si256(q.as_mut_ptr().add(i).cast(), rv); - i += 4; + i = i.strict_add(4); } // Row pass: 8 P-rounds on contiguous 16-u64 chunks of q[]. let mut row = 0usize; while row < 8 { - let base = row * 16; + let base = row.strict_mul(16); let mut a = _mm256_loadu_si256(q.as_ptr().add(base).cast()); - let mut b = _mm256_loadu_si256(q.as_ptr().add(base + 4).cast()); - let mut c = _mm256_loadu_si256(q.as_ptr().add(base + 8).cast()); - let mut d = _mm256_loadu_si256(q.as_ptr().add(base + 12).cast()); + let mut b = _mm256_loadu_si256(q.as_ptr().add(base.strict_add(4)).cast()); + let mut c = _mm256_loadu_si256(q.as_ptr().add(base.strict_add(8)).cast()); + let mut d = _mm256_loadu_si256(q.as_ptr().add(base.strict_add(12)).cast()); p_round_avx2(&mut a, &mut b, &mut c, &mut d); _mm256_storeu_si256(q.as_mut_ptr().add(base).cast(), a); - _mm256_storeu_si256(q.as_mut_ptr().add(base + 4).cast(), b); - _mm256_storeu_si256(q.as_mut_ptr().add(base + 8).cast(), c); - _mm256_storeu_si256(q.as_mut_ptr().add(base + 12).cast(), d); - row += 1; + _mm256_storeu_si256(q.as_mut_ptr().add(base.strict_add(4)).cast(), b); + _mm256_storeu_si256(q.as_mut_ptr().add(base.strict_add(8)).cast(), c); + _mm256_storeu_si256(q.as_mut_ptr().add(base.strict_add(12)).cast(), d); + row = row.strict_add(1); } // Column pass: 8 P-rounds on stride-16 u64 sequences. Each YMM holds @@ -126,19 +125,19 @@ pub(super) unsafe fn compress_avx2( // high half = row 2k+1) — see RFC 9106 §3.6 column-step indexing. let mut col = 0usize; while col < 8 { - let base = col * 2; - let mut a = load_col_pair_avx2(&q, base, base + 16); - let mut b = load_col_pair_avx2(&q, base + 32, base + 48); - let mut c = load_col_pair_avx2(&q, base + 64, base + 80); - let mut d = load_col_pair_avx2(&q, base + 96, base + 112); + let base = col.strict_mul(2); + let mut a = load_col_pair_avx2(&q, base, base.strict_add(16)); + let mut b = load_col_pair_avx2(&q, base.strict_add(32), base.strict_add(48)); + let mut c = load_col_pair_avx2(&q, base.strict_add(64), base.strict_add(80)); + let mut d = load_col_pair_avx2(&q, base.strict_add(96), base.strict_add(112)); p_round_avx2(&mut a, &mut b, &mut c, &mut d); - store_col_pair_avx2(&mut q, base, base + 16, a); - store_col_pair_avx2(&mut q, base + 32, base + 48, b); - store_col_pair_avx2(&mut q, base + 64, base + 80, c); - store_col_pair_avx2(&mut q, base + 96, base + 112, d); - col += 1; + store_col_pair_avx2(&mut q, base, base.strict_add(16), a); + store_col_pair_avx2(&mut q, base.strict_add(32), base.strict_add(48), b); + store_col_pair_avx2(&mut q, base.strict_add(64), base.strict_add(80), c); + store_col_pair_avx2(&mut q, base.strict_add(96), base.strict_add(112), d); + col = col.strict_add(1); } // Final XOR with R, fused with the dst store/xor. @@ -153,7 +152,7 @@ pub(super) unsafe fn compress_avx2( } else { _mm256_storeu_si256(dst.as_mut_ptr().add(i).cast(), f); } - i += 4; + i = i.strict_add(4); } } } @@ -251,12 +250,18 @@ unsafe fn gb_avx2(a: &mut __m256i, b: &mut __m256i, c: &mut __m256i, d: &mut __m } #[inline(always)] +/// # Safety +/// +/// The current CPU must support AVX2. unsafe fn ror32_avx2(x: __m256i) -> __m256i { // SAFETY: AVX2 inherited; shuffle imm 0xB1 swaps adjacent u32 halves. unsafe { _mm256_shuffle_epi32(x, 0xB1) } } #[inline(always)] +/// # Safety +/// +/// The current CPU must support AVX2. unsafe fn ror24_avx2(x: __m256i) -> __m256i { // SAFETY: AVX2 inherited; ROT24_MASK is 32-byte aligned static data. unsafe { @@ -266,6 +271,9 @@ unsafe fn ror24_avx2(x: __m256i) -> __m256i { } #[inline(always)] +/// # Safety +/// +/// The current CPU must support AVX2. unsafe fn ror16_avx2(x: __m256i) -> __m256i { // SAFETY: AVX2 inherited; ROT16_MASK is 32-byte aligned static data. unsafe { @@ -275,6 +283,9 @@ unsafe fn ror16_avx2(x: __m256i) -> __m256i { } #[inline(always)] +/// # Safety +/// +/// The current CPU must support AVX2. unsafe fn ror63_avx2(x: __m256i) -> __m256i { // SAFETY: AVX2 inherited; (x << 1) | (x >> 63). unsafe { _mm256_or_si256(_mm256_add_epi64(x, x), _mm256_srli_epi64(x, 63)) } @@ -319,7 +330,7 @@ pub(super) unsafe fn compress_avx512( let rv = _mm512_xor_si512(xv, yv); _mm512_storeu_si512(r.as_mut_ptr().add(i).cast(), rv); _mm512_storeu_si512(q.as_mut_ptr().add(i).cast(), rv); - i += 8; + i = i.strict_add(8); } // Row pass: 4 iterations × 2 rows per iter = 8 P-rounds done at @@ -339,14 +350,14 @@ pub(super) unsafe fn compress_avx512( // Each `VSHUFI64X2` picks 4 of 8 contiguous 128-bit lanes (4 from // src1, 4 from src2) using a single imm8. The 0x44 imm picks lanes // 0,1 from each source; 0xEE picks lanes 2,3. - let mut iter = 0; + let mut iter = 0usize; while iter < 4 { - let off = iter * 32; + let off = iter.strict_mul(32); let r0_lo = _mm512_loadu_si512(q.as_ptr().add(off).cast()); - let r0_hi = _mm512_loadu_si512(q.as_ptr().add(off + 8).cast()); - let r1_lo = _mm512_loadu_si512(q.as_ptr().add(off + 16).cast()); - let r1_hi = _mm512_loadu_si512(q.as_ptr().add(off + 24).cast()); + let r0_hi = _mm512_loadu_si512(q.as_ptr().add(off.strict_add(8)).cast()); + let r1_lo = _mm512_loadu_si512(q.as_ptr().add(off.strict_add(16)).cast()); + let r1_hi = _mm512_loadu_si512(q.as_ptr().add(off.strict_add(24)).cast()); let mut a = _mm512_shuffle_i64x2(r0_lo, r1_lo, 0x44); let mut b = _mm512_shuffle_i64x2(r0_lo, r1_lo, 0xEE); @@ -363,10 +374,10 @@ pub(super) unsafe fn compress_avx512( let r1_hi_out = _mm512_shuffle_i64x2(c, d, 0xEE); _mm512_storeu_si512(q.as_mut_ptr().add(off).cast(), r0_lo_out); - _mm512_storeu_si512(q.as_mut_ptr().add(off + 8).cast(), r0_hi_out); - _mm512_storeu_si512(q.as_mut_ptr().add(off + 16).cast(), r1_lo_out); - _mm512_storeu_si512(q.as_mut_ptr().add(off + 24).cast(), r1_hi_out); - iter += 1; + _mm512_storeu_si512(q.as_mut_ptr().add(off.strict_add(8)).cast(), r0_hi_out); + _mm512_storeu_si512(q.as_mut_ptr().add(off.strict_add(16)).cast(), r1_lo_out); + _mm512_storeu_si512(q.as_mut_ptr().add(off.strict_add(24)).cast(), r1_hi_out); + iter = iter.strict_add(1); } // Column pass: 4-way YMM with native `VPRORQ` rotations. @@ -379,19 +390,19 @@ pub(super) unsafe fn compress_avx512( // 2 × insert per GB-lane. let mut col = 0usize; while col < 8 { - let base = col * 2; - let mut a = load_col_pair_avx2(&q, base, base + 16); - let mut b = load_col_pair_avx2(&q, base + 32, base + 48); - let mut c = load_col_pair_avx2(&q, base + 64, base + 80); - let mut d = load_col_pair_avx2(&q, base + 96, base + 112); + let base = col.strict_mul(2); + let mut a = load_col_pair_avx2(&q, base, base.strict_add(16)); + let mut b = load_col_pair_avx2(&q, base.strict_add(32), base.strict_add(48)); + let mut c = load_col_pair_avx2(&q, base.strict_add(64), base.strict_add(80)); + let mut d = load_col_pair_avx2(&q, base.strict_add(96), base.strict_add(112)); p_round_avx512vl(&mut a, &mut b, &mut c, &mut d); - store_col_pair_avx2(&mut q, base, base + 16, a); - store_col_pair_avx2(&mut q, base + 32, base + 48, b); - store_col_pair_avx2(&mut q, base + 64, base + 80, c); - store_col_pair_avx2(&mut q, base + 96, base + 112, d); - col += 1; + store_col_pair_avx2(&mut q, base, base.strict_add(16), a); + store_col_pair_avx2(&mut q, base.strict_add(32), base.strict_add(48), b); + store_col_pair_avx2(&mut q, base.strict_add(64), base.strict_add(80), c); + store_col_pair_avx2(&mut q, base.strict_add(96), base.strict_add(112), d); + col = col.strict_add(1); } // Final XOR with R, fused with dst store/xor at ZMM width. @@ -406,7 +417,7 @@ pub(super) unsafe fn compress_avx512( } else { _mm512_storeu_si512(dst.as_mut_ptr().add(i).cast(), f); } - i += 8; + i = i.strict_add(8); } } } diff --git a/src/auth/curve25519_edwards.rs b/src/auth/curve25519_edwards.rs index 48b65cc4..4fc7eb72 100644 --- a/src/auth/curve25519_edwards.rs +++ b/src/auth/curve25519_edwards.rs @@ -4,28 +4,23 @@ //! same Curve25519 basepoint. Reuse the Ed25519 basepoint machinery instead of //! maintaining a second precompute stack for the identical scalar-mul problem. -#[allow(dead_code)] +#[cfg(feature = "ed25519")] #[path = "ed25519/constants.rs"] pub(crate) mod constants; -#[allow(dead_code)] #[path = "ed25519/field.rs"] pub(crate) mod field; #[cfg(target_arch = "x86_64")] -#[allow(dead_code)] #[path = "ed25519/field_avx2.rs"] pub(crate) mod field_avx2; #[cfg(target_arch = "x86_64")] -#[allow(dead_code)] #[path = "ed25519/field_ifma.rs"] pub(crate) mod field_ifma; -#[allow(dead_code)] #[path = "ed25519/point.rs"] pub(crate) mod point; #[cfg(target_arch = "x86_64")] -#[allow(dead_code)] #[path = "ed25519/point_avx2.rs"] pub(crate) mod point_avx2; -#[allow(dead_code)] +#[cfg(feature = "ed25519")] #[path = "ed25519/scalar.rs"] pub(crate) mod scalar; @@ -39,7 +34,6 @@ pub use point_avx2::{ /// Dispatch `[s]B` (fixed-base scalar mul) to the fastest validated CT path. #[must_use] -#[allow(dead_code)] pub(crate) fn basepoint_mul_dispatch(scalar_bytes: &[u8; 32]) -> point::ExtendedPoint { #[cfg(target_arch = "x86_64")] { @@ -59,3 +53,26 @@ pub(crate) fn basepoint_mul_dispatch(scalar_bytes: &[u8; 32]) -> point::Extended point::ExtendedPoint::scalar_mul_basepoint(scalar_bytes) } + +/// Decompose a scalar encoding into signed radix-16 digits in `[-8, 8]`. +#[must_use] +fn scalar_radix_16(bytes: &[u8; 32]) -> [i8; 64] { + debug_assert!(bytes[31] <= 127); + + let mut digits = [0i8; 64]; + for (index, byte) in bytes.iter().copied().enumerate() { + let low = index.strict_mul(2); + let high = low.strict_add(1); + digits[low] = i8::from_ne_bytes([byte & 0x0F]); + digits[high] = i8::from_ne_bytes([(byte >> 4) & 0x0F]); + } + + for index in 0usize..63 { + let next = index.strict_add(1); + let carry = digits[index].strict_add(8) >> 4; + digits[index] = digits[index].strict_sub(carry << 4); + digits[next] = digits[next].strict_add(carry); + } + + digits +} diff --git a/src/auth/ecdsa.rs b/src/auth/ecdsa.rs index 5ebffd1d..0c701e0e 100644 --- a/src/auth/ecdsa.rs +++ b/src/auth/ecdsa.rs @@ -21,6 +21,13 @@ use crate::{ mod ecdsa_aarch64_asm; #[path = "ecdsa_generator_tables.rs"] mod ecdsa_generator_tables; +#[cfg(any( + test, + not(any( + all(target_arch = "aarch64", any(target_os = "macos", target_os = "linux")), + all(target_arch = "x86_64", target_os = "linux") + )) +))] #[path = "ecdsa_p384_field.rs"] mod ecdsa_p384_field; #[cfg(all(target_arch = "x86_64", target_os = "linux"))] @@ -1609,23 +1616,20 @@ fn ct_mul_u64_wide(lhs: u64, rhs: u64) -> (u64, u64) { let mut multiplicand_lo = lhs; let mut multiplicand_hi = 0u64; let mut multiplier = rhs; - let mut bit = 0u32; - - while bit < u64::BITS { + for _ in 0..u64::BITS { // Keep LLVM from recognizing the bit-serial product and lowering it back to a target multiply. // The CT artifact gate independently rejects scalar multiply in these ECDSA closures. let selected_bit = core::hint::black_box(multiplier & 1); let mask = 0u64.wrapping_sub(selected_bit); let (next_lo, carry) = product_lo.overflowing_add(multiplicand_lo & mask); let (next_hi, _) = product_hi.overflowing_add(multiplicand_hi & mask); - let (next_hi, _) = next_hi.overflowing_add(carry as u64); + let (next_hi, _) = next_hi.overflowing_add(u64::from(carry)); product_lo = next_lo; product_hi = next_hi; multiplicand_hi = (multiplicand_hi << 1) | (multiplicand_lo >> 63); multiplicand_lo <<= 1; multiplier >>= 1; - bit += 1; } (product_lo, product_hi) @@ -1633,26 +1637,25 @@ fn ct_mul_u64_wide(lhs: u64, rhs: u64) -> (u64, u64) { #[inline(always)] fn mul_u64_wide(lhs: u64, rhs: u64) -> (u64, u64) { - #[cfg(any(test, target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x"))] + #[cfg(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x"))] { ct_mul_u64_wide(lhs, rhs) } - #[cfg(not(any(test, target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x")))] + #[cfg(not(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x")))] { - let product = (lhs as u128) * (rhs as u128); - (product as u64, (product >> 64) as u64) + split_u128(u128::from(lhs).strict_mul(u128::from(rhs))) } } #[inline(always)] fn mul_u64_low(lhs: u64, rhs: u64) -> u64 { - #[cfg(any(test, target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x"))] + #[cfg(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x"))] { ct_mul_u64_wide(lhs, rhs).0 } - #[cfg(not(any(test, target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x")))] + #[cfg(not(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x")))] { lhs.wrapping_mul(rhs) } @@ -1714,17 +1717,15 @@ impl Uint { } fn from_be_slice(bytes: &[u8]) -> Result { - if bytes.len() > L * 8 { + if bytes.len() > L.strict_mul(8) { return Err(EcdsaError::InvalidSignature); } let mut out = Self::ZERO; - for (index, byte) in bytes.iter().rev().copied().enumerate() { - let limb = index / 8; - let shift = (index % 8) * 8; - let Some(out_limb) = out.0.get_mut(limb) else { - return Err(EcdsaError::InvalidSignature); - }; - *out_limb |= u64::from(byte) << shift; + for (out_limb, chunk) in out.0.iter_mut().zip(bytes.rchunks(8)) { + let mut encoded = [0u8; 8]; + let offset = encoded.len().strict_sub(chunk.len()); + encoded[offset..].copy_from_slice(chunk); + *out_limb = u64::from_be_bytes(encoded); } Ok(out) } @@ -1739,12 +1740,10 @@ impl Uint { fn write_be(self, out: &mut [u8]) { out.fill(0); - for (index, byte) in out.iter_mut().rev().enumerate() { - let limb = index / 8; - let shift = (index % 8) * 8; - if let Some(limb_value) = self.0.get(limb) { - *byte = (*limb_value >> shift) as u8; - } + for (chunk, limb) in out.rchunks_mut(8).zip(self.0) { + let encoded = limb.to_be_bytes(); + let offset = encoded.len().strict_sub(chunk.len()); + chunk.copy_from_slice(&encoded[offset..]); } } @@ -1948,24 +1947,18 @@ impl Uint { } const WINDOW_BITS: usize = 4; - const WINDOW_SIZE: usize = 1 << WINDOW_BITS; + const WINDOW_SIZE: usize = 16; let mut base = montgomery_mul(*self, modulus.r2, modulus); let mut acc = montgomery_mul(Self::ONE, modulus.r2, modulus); let mut powers = [acc; WINDOW_SIZE]; - let mut next_power = base; - let mut remaining = WINDOW_SIZE - 1; + let mut next_power = acc; for power in powers.iter_mut().skip(1) { + next_power = montgomery_mul(next_power, base, modulus); *power = next_power; - remaining -= 1; - if remaining != 0 { - next_power = montgomery_mul(next_power, base, modulus); - } } - let mut bit = L * 64; - while bit > 0 { - bit -= WINDOW_BITS; + for bit in (0..L.strict_mul(64)).step_by(WINDOW_BITS).rev() { for _ in 0..WINDOW_BITS { acc = montgomery_square(acc, modulus); } @@ -1999,15 +1992,14 @@ impl SecretScalar { } fn from_be_bytes(bytes: &[u8; N]) -> Self { - debug_assert!(N <= L * 8); + debug_assert!(N <= L.strict_mul(8)); let mut out = Uint::ZERO; - for (index, byte) in bytes.iter().rev().copied().enumerate() { - let limb = index / 8; - let shift = (index % 8) * 8; - if let Some(out_limb) = out.0.get_mut(limb) { - *out_limb |= u64::from(byte) << shift; - } + for (out_limb, chunk) in out.0.iter_mut().zip(bytes.rchunks(8)) { + let mut encoded = [0u8; 8]; + let offset = encoded.len().strict_sub(chunk.len()); + encoded[offset..].copy_from_slice(chunk); + *out_limb = u64::from_be_bytes(encoded); } Self::new(out) } @@ -2068,7 +2060,6 @@ impl FieldElement { Self::from_uint(Uint::ONE, modulus) } - #[allow(clippy::indexing_slicing)] fn add(self, rhs: Self) -> Self { if is_p384_field_modulus(self.modulus) { let reduced = add_p384_field( @@ -2096,7 +2087,6 @@ impl FieldElement { Self::from_montgomery(self.value.add_mod_ct(&rhs.value, self.modulus.value), self.modulus) } - #[allow(clippy::indexing_slicing)] fn sub(self, rhs: Self) -> Self { if is_p384_field_modulus(self.modulus) { let reduced = sub_p384_field( @@ -2360,6 +2350,11 @@ impl Jacobian { } } + #[cfg(any( + feature = "diag", + all(target_arch = "aarch64", any(target_os = "macos", target_os = "linux")), + all(target_arch = "x86_64", target_os = "linux") + ))] fn to_affine_x_ct(self, exponent: Uint) -> FieldElement { let inv_z = self.z.inv_ct(exponent); let z2 = inv_z.square(); @@ -2492,39 +2487,47 @@ fn sign_digest_p384_blinded( } fn hmac_expand_p256(secret: &[u8; 32], digest: &[u8; 32], out: &mut [u8; 64]) { - for (block_index, block) in out.chunks_exact_mut(HmacSha256::TAG_SIZE).enumerate() { + let (blocks, remainder) = out.as_chunks_mut::<{ HmacSha256::TAG_SIZE }>(); + debug_assert!(remainder.is_empty()); + for (block_index, block) in [0u8, 1].into_iter().zip(blocks) { let mut mac = HmacSha256::new(secret); mac.update(P256_NONCE_DOMAIN); - mac.update(&[block_index as u8]); + mac.update(&[block_index]); mac.update(digest); block.copy_from_slice(mac.finalize().as_bytes()); } } fn hmac_expand_p256_public_blind(secret: &[u8; 32], out: &mut [u8; 64]) { - for (block_index, block) in out.chunks_exact_mut(HmacSha256::TAG_SIZE).enumerate() { + let (blocks, remainder) = out.as_chunks_mut::<{ HmacSha256::TAG_SIZE }>(); + debug_assert!(remainder.is_empty()); + for (block_index, block) in [0u8, 1].into_iter().zip(blocks) { let mut mac = HmacSha256::new(secret); mac.update(P256_PUBKEY_BLIND_DOMAIN); - mac.update(&[block_index as u8]); + mac.update(&[block_index]); block.copy_from_slice(mac.finalize().as_bytes()); } } fn hmac_expand_p384(secret: &[u8; 48], digest: &[u8; 48], out: &mut [u8; 96]) { - for (block_index, block) in out.chunks_exact_mut(HmacSha384::TAG_SIZE).enumerate() { + let (blocks, remainder) = out.as_chunks_mut::<{ HmacSha384::TAG_SIZE }>(); + debug_assert!(remainder.is_empty()); + for (block_index, block) in [0u8, 1].into_iter().zip(blocks) { let mut mac = HmacSha384::new(secret); mac.update(P384_NONCE_DOMAIN); - mac.update(&[block_index as u8]); + mac.update(&[block_index]); mac.update(digest); block.copy_from_slice(mac.finalize().as_bytes()); } } fn hmac_expand_p384_public_blind(secret: &[u8; 48], out: &mut [u8; 96]) { - for (block_index, block) in out.chunks_exact_mut(HmacSha384::TAG_SIZE).enumerate() { + let (blocks, remainder) = out.as_chunks_mut::<{ HmacSha384::TAG_SIZE }>(); + debug_assert!(remainder.is_empty()); + for (block_index, block) in [0u8, 1].into_iter().zip(blocks) { let mut mac = HmacSha384::new(secret); mac.update(P384_PUBKEY_BLIND_DOMAIN); - mac.update(&[block_index as u8]); + mac.update(&[block_index]); block.copy_from_slice(mac.finalize().as_bytes()); } } @@ -2755,7 +2758,9 @@ fn reduce_wide_order_nonzero_owned( } let mut acc = Uint::ZERO; - for chunk in bytes.chunks_exact(8) { + let (chunks, remainder) = bytes.as_chunks::<8>(); + debug_assert!(remainder.is_empty()); + for chunk in chunks { acc = mul_mod_montgomery_ct(acc, radix, modulus); let word = chunk .iter() @@ -2946,12 +2951,12 @@ fn is_p384_curve(curve: &Curve) -> bool { all(target_arch = "aarch64", any(target_os = "macos", target_os = "linux")), all(target_arch = "x86_64", target_os = "linux") ))] -#[allow(clippy::indexing_slicing)] fn affine_from_words(modulus: &'static Modulus, words: &[u64]) -> Affine { let mut x = [0u64; L]; let mut y = [0u64; L]; - x.copy_from_slice(&words[..L]); - y.copy_from_slice(&words[L..L * 2]); + let (x_words, y_words) = words.split_at(L); + x.copy_from_slice(x_words); + y.copy_from_slice(y_words); let x = Uint(x); let y = Uint(y); Affine { @@ -3276,14 +3281,13 @@ fn p384_jacobian_from_words(words: &[u64; 18]) -> Jacobian<6> { } #[cfg(all(target_arch = "aarch64", any(target_os = "macos", target_os = "linux")))] -#[allow(clippy::indexing_slicing)] fn jacobian_from_p384_words(modulus: &'static Modulus, words: &[u64; 18]) -> Jacobian { let mut x = [0u64; L]; let mut y = [0u64; L]; let mut z = [0u64; L]; x.copy_from_slice(&words[..L]); - y.copy_from_slice(&words[6..6 + L]); - z.copy_from_slice(&words[12..12 + L]); + y.copy_from_slice(&words[6..6usize.strict_add(L)]); + z.copy_from_slice(&words[12..12usize.strict_add(L)]); Jacobian { x: FieldElement::from_montgomery(Uint(x), modulus), y: FieldElement::from_montgomery(Uint(y), modulus), @@ -3293,7 +3297,6 @@ fn jacobian_from_p384_words(modulus: &'static Modulus, words: } #[cfg(test)] -#[allow(clippy::indexing_slicing)] fn scalar_mul_basepoint_comb_ct(curve: &Curve, scalar: Uint) -> Jacobian { let rows = curve.signing_comb_rows; let mut acc = Jacobian::infinity(curve.field_modulus); @@ -3308,7 +3311,6 @@ fn scalar_mul_basepoint_comb_ct(curve: &Curve, scalar: Uint( curve: &Curve, scalar: &SecretScalar, @@ -3327,7 +3329,6 @@ fn scalar_mul_basepoint_comb_ct_secret( acc } -#[allow(clippy::indexing_slicing)] fn scalar_mul_basepoint_comb_ct_secret_blinded( curve: &Curve, scalar: &SecretScalar, @@ -3373,7 +3374,6 @@ fn signing_comb_digit_ct(scalar: Uint, row: usize, rows: usiz digit } -#[allow(clippy::indexing_slicing)] fn select_signing_generator_affine_ct(curve: &Curve, digit: usize) -> Affine { let mut x = curve.signing_generator_comb_x[0]; let mut y = curve.signing_generator_comb_y[0]; @@ -3393,6 +3393,7 @@ fn select_signing_generator_affine_ct(curve: &Curve, digit: u } } +/// Return the P-256 signing-comb coordinates selected by `digit` as Montgomery limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_select_signing_generator_affine_limb_digest(digit: u8) -> [u64; 8] { let selected = select_signing_generator_affine_ct(&P256, usize::from(digit)); @@ -3424,6 +3425,7 @@ pub(crate) fn diag_zeroize_ecdsa_p256_platform_scratch(wide: [u8; 64]) -> u64 { core::hint::black_box(inverse.value().0[0]) } +/// Derive the deterministic P-256 nonce for `message` and return its scalar limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_nonce_reduce_limb_digest(secret: [u8; 32], message: &[u8]) -> [u64; 4] { let secret = ZeroizingBytes::new(secret); @@ -3434,6 +3436,7 @@ pub fn diag_ecdsa_p256_nonce_reduce_limb_digest(secret: [u8; 32], message: &[u8] nonce.value().0 } +/// Reduce a wide P-256 nonce candidate to a nonzero scalar and return its limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_reduce_wide_order_limb_digest(wide: [u8; 64]) -> [u64; 4] { let wide = ZeroizingBytes::new(wide); @@ -3441,6 +3444,7 @@ pub fn diag_ecdsa_p256_reduce_wide_order_limb_digest(wide: [u8; 64]) -> [u64; 4] nonce.value().0 } +/// Return the affine limbs produced by blinded P-256 basepoint multiplication. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_basepoint_blinded_limb_digest(secret: [u8; 32], blind: [u8; 64], message: &[u8]) -> [u64; 8] { let secret = ZeroizingBytes::new(secret); @@ -3457,6 +3461,7 @@ pub fn diag_ecdsa_p256_basepoint_blinded_limb_digest(secret: [u8; 32], blind: [u out } +/// Run P-256 scalar signing finalization with supplied nonce material and return `r || s` limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_scalar_finish_limb_digest(secret: [u8; 32], nonce_wide: [u8; 64], message: &[u8]) -> [u64; 8] { let secret = ZeroizingBytes::new(secret); @@ -3472,6 +3477,7 @@ pub fn diag_ecdsa_p256_scalar_finish_limb_digest(secret: [u8; 32], nonce_wide: [ out } +/// Multiply the P-256 secret scalar by a fixed public `r` and return the order-field limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_order_mul_fixed_r_limb_digest(secret: [u8; 32]) -> [u64; 4] { let secret = ZeroizingBytes::new(secret); @@ -3481,6 +3487,7 @@ pub fn diag_ecdsa_p256_order_mul_fixed_r_limb_digest(secret: [u8; 32]) -> [u64; rd.value().0 } +/// Run the blinded P-256 order multiplication stage for a fixed public `r` and return its limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_order_mul_blinded_fixed_r_limb_digest(secret: [u8; 32], blind: [u8; 64]) -> [u64; 4] { let secret = ZeroizingBytes::new(secret); @@ -3497,6 +3504,7 @@ pub fn diag_ecdsa_p256_order_mul_blinded_fixed_r_limb_digest(secret: [u8; 32], b rd.value().0 } +/// Derive and invert the deterministic P-256 nonce and return its Montgomery limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_nonce_inverse_limb_digest(secret: [u8; 32], message: &[u8]) -> [u64; 4] { let secret = ZeroizingBytes::new(secret); @@ -3512,6 +3520,7 @@ pub fn diag_ecdsa_p256_nonce_inverse_limb_digest(secret: [u8; 32], message: &[u8 inverse.value().0 } +/// Run the final P-256 signing multiplication with supplied nonce material and return its limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_final_multiply_limb_digest(secret: [u8; 32], nonce_wide: [u8; 64], message: &[u8]) -> [u64; 4] { let secret = ZeroizingBytes::new(secret); @@ -3536,6 +3545,7 @@ pub fn diag_ecdsa_p256_final_multiply_limb_digest(secret: [u8; 32], nonce_wide: product.value().0 } +/// Return the P-384 signing-comb coordinates selected by `digit` as Montgomery limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_select_signing_generator_affine_limb_digest(digit: u8) -> [u64; 12] { let selected = select_signing_generator_affine_ct(&P384, usize::from(digit)); @@ -3565,6 +3575,7 @@ pub(crate) fn diag_zeroize_ecdsa_p384_platform_scratch(wide: [u8; 96]) -> u64 { core::hint::black_box(inverse.value().0[0]) } +/// Derive the deterministic P-384 nonce for `message` and return its scalar limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_nonce_reduce_limb_digest(secret: [u8; 48], message: &[u8]) -> [u64; 6] { let secret = ZeroizingBytes::new(secret); @@ -3575,6 +3586,7 @@ pub fn diag_ecdsa_p384_nonce_reduce_limb_digest(secret: [u8; 48], message: &[u8] nonce.value().0 } +/// Reduce a wide P-384 nonce candidate to a nonzero scalar and return its limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_reduce_wide_order_limb_digest(wide: [u8; 96]) -> [u64; 6] { let wide = ZeroizingBytes::new(wide); @@ -3582,6 +3594,7 @@ pub fn diag_ecdsa_p384_reduce_wide_order_limb_digest(wide: [u8; 96]) -> [u64; 6] nonce.value().0 } +/// Return the affine limbs produced by blinded P-384 basepoint multiplication. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_basepoint_blinded_limb_digest(secret: [u8; 48], blind: [u8; 96], message: &[u8]) -> [u64; 12] { let secret = ZeroizingBytes::new(secret); @@ -3598,6 +3611,7 @@ pub fn diag_ecdsa_p384_basepoint_blinded_limb_digest(secret: [u8; 48], blind: [u out } +/// Derive the P-384 nonce point and return its reduced affine x-coordinate limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_basepoint_r_limb_digest(secret: [u8; 48], message: &[u8]) -> [u64; 6] { let secret = ZeroizingBytes::new(secret); @@ -3614,6 +3628,7 @@ pub fn diag_ecdsa_p384_basepoint_r_limb_digest(secret: [u8; 48], message: &[u8]) .0 } +/// Run P-384 scalar signing finalization with supplied nonce material and return `r || s` limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_scalar_finish_limb_digest(secret: [u8; 48], nonce_wide: [u8; 96], message: &[u8]) -> [u64; 12] { let secret = ZeroizingBytes::new(secret); @@ -3629,6 +3644,7 @@ pub fn diag_ecdsa_p384_scalar_finish_limb_digest(secret: [u8; 48], nonce_wide: [ out } +/// Multiply the P-384 secret scalar by a fixed public `r` and return the order-field limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_order_mul_fixed_r_limb_digest(secret: [u8; 48]) -> [u64; 6] { let secret = ZeroizingBytes::new(secret); @@ -3638,6 +3654,7 @@ pub fn diag_ecdsa_p384_order_mul_fixed_r_limb_digest(secret: [u8; 48]) -> [u64; rd.value().0 } +/// Derive and invert the deterministic P-384 nonce and return its Montgomery limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_nonce_inverse_limb_digest(secret: [u8; 48], message: &[u8]) -> [u64; 6] { let secret = ZeroizingBytes::new(secret); @@ -3653,6 +3670,7 @@ pub fn diag_ecdsa_p384_nonce_inverse_limb_digest(secret: [u8; 48], message: &[u8 inverse.value().0 } +/// Run the final P-384 signing multiplication with supplied nonce material and return its limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_final_multiply_limb_digest(secret: [u8; 48], nonce_wide: [u8; 96], message: &[u8]) -> [u64; 6] { let secret = ZeroizingBytes::new(secret); @@ -3697,7 +3715,6 @@ fn projective_x_matches_scalar(point: Jacobian, scalar: Uint< } } -#[allow(clippy::indexing_slicing)] fn scalar_mul_two( curve: &Curve, lhs_scalar: Uint, @@ -3723,13 +3740,13 @@ fn scalar_mul_two( } const fn comb_rows() -> usize { - (L * 64).div_ceil(COMB_WIDTH) + L.strict_mul(64).div_ceil(COMB_WIDTH) } fn comb_digit(scalar: Uint, row: usize, rows: usize) -> usize { let mut digit = 0usize; for column in 0..COMB_WIDTH { - let bit = row + column * rows; + let bit = row.strict_add(column.strict_mul(rows)); if scalar.bit(bit) { digit |= 1usize << column; } @@ -3737,7 +3754,6 @@ fn comb_digit(scalar: Uint, row: usize, rows: usize) -> usize digit } -#[allow(clippy::indexing_slicing)] fn precompute_comb_table(point: Affine) -> [Affine; COMB_TABLE_SIZE] { let rows = comb_rows::(); let mut column_points = [Jacobian::from_affine(point); COMB_WIDTH]; @@ -3763,14 +3779,16 @@ fn precompute_comb_table(point: Affine) -> [Affine; COMB_T normalize_jacobian_table(table) } -#[allow(clippy::indexing_slicing)] fn normalize_jacobian_table(table: [Jacobian; N]) -> [Affine; N] { - let modulus = table[0].x.modulus; + let modulus = table + .first() + .map(|point| point.x.modulus) + .expect("ECDSA comb tables are nonempty"); let mut prefixes = [FieldElement::one(modulus); N]; let mut acc = FieldElement::one(modulus); - for i in 0..N { - prefixes[i] = acc; - acc = acc.mul(table[i].z); + for (prefix, point) in prefixes.iter_mut().zip(table.iter()) { + *prefix = acc; + acc = acc.mul(point.z); } let mut acc_inv = acc.inv(); @@ -3778,30 +3796,29 @@ fn normalize_jacobian_table(table: [Jacobian; x: FieldElement::zero(modulus), y: FieldElement::zero(modulus), }; N]; - for i in (0..N).rev() { - let z_inv = acc_inv.mul(prefixes[i]); - acc_inv = acc_inv.mul(table[i].z); + for ((point, prefix), output) in table.iter().zip(prefixes).zip(out.iter_mut()).rev() { + let z_inv = acc_inv.mul(prefix); + acc_inv = acc_inv.mul(point.z); let z2 = z_inv.square(); let z3 = z2.mul(z_inv); - out[i] = Affine { - x: table[i].x.mul(z2), - y: table[i].y.mul(z3), + *output = Affine { + x: point.x.mul(z2), + y: point.y.mul(z3), }; } out } -#[allow(clippy::indexing_slicing)] fn parse_public_key(bytes: &[u8], curve: &Curve) -> Result, EcdsaError> { - let field_len = L * 8; + let field_len = L.strict_mul(8); if bytes.len() != field_len.strict_mul(2).strict_add(1) || bytes.first().copied() != Some(0x04) { return Err(EcdsaError::InvalidPublicKey); } - let x = Uint::from_be_slice(bytes.get(1..1 + field_len).ok_or(EcdsaError::InvalidPublicKey)?) - .map_err(|_| EcdsaError::InvalidPublicKey)?; - let y = Uint::from_be_slice(bytes.get(1 + field_len..).ok_or(EcdsaError::InvalidPublicKey)?) - .map_err(|_| EcdsaError::InvalidPublicKey)?; + let coordinates = bytes.get(1..).ok_or(EcdsaError::InvalidPublicKey)?; + let (x_bytes, y_bytes) = coordinates.split_at(field_len); + let x = Uint::from_be_slice(x_bytes).map_err(|_| EcdsaError::InvalidPublicKey)?; + let y = Uint::from_be_slice(y_bytes).map_err(|_| EcdsaError::InvalidPublicKey)?; if x.cmp(&curve.field_modulus.value).is_ge() || y.cmp(&curve.field_modulus.value).is_ge() { return Err(EcdsaError::InvalidPublicKey); } @@ -3827,10 +3844,11 @@ fn is_on_curve(point: Affine, curve: &Curve) -> bool { fn encode_sec1(point: &Affine) -> [u8; N] { let mut out = [0u8; N]; - out[0] = 0x04; - let field_len = L * 8; - point.x.to_uint().write_be(&mut out[1..1 + field_len]); - point.y.to_uint().write_be(&mut out[1 + field_len..]); + let (tag, coordinates) = out.split_first_mut().expect("SEC1 output includes a tag byte"); + *tag = 0x04; + let (x_bytes, y_bytes) = coordinates.split_at_mut(L.strict_mul(8)); + point.x.to_uint().write_be(x_bytes); + point.y.to_uint().write_be(y_bytes); out } @@ -3838,9 +3856,9 @@ fn parse_signature_scalars( bytes: &[u8; BYTES], scalar_modulus: Uint, ) -> Result<(Uint, Uint), EcdsaError> { - let field_len = LIMBS * 8; - let r = Uint::from_be_slice(&bytes[..field_len])?; - let s = Uint::from_be_slice(&bytes[field_len..])?; + let (r_bytes, s_bytes) = bytes.split_at(LIMBS.strict_mul(8)); + let r = Uint::from_be_slice(r_bytes)?; + let s = Uint::from_be_slice(s_bytes)?; if !r.is_in_range(&scalar_modulus) || !s.is_in_range(&scalar_modulus) { return Err(EcdsaError::InvalidSignature); } @@ -3848,7 +3866,7 @@ fn parse_signature_scalars( } fn parse_signature_der_bytes(der: &[u8]) -> Result<[u8; BYTES], EcdsaError> { - let field_len = LIMBS * 8; + let field_len = LIMBS.strict_mul(8); let mut root = DerReader::new(der); let sig = root.read_constructed(TAG_SEQUENCE)?; root.finish()?; @@ -3859,8 +3877,9 @@ fn parse_signature_der_bytes(der: &[u8]) sig.finish()?; let mut bytes = [0u8; BYTES]; - parse_der_integer_into(r_value, &mut bytes[..field_len])?; - parse_der_integer_into(s_value, &mut bytes[field_len..])?; + let (r_bytes, s_bytes) = bytes.split_at_mut(field_len); + parse_der_integer_into(r_value, r_bytes)?; + parse_der_integer_into(s_value, s_bytes)?; Ok(bytes) } @@ -3959,7 +3978,6 @@ fn mul_mod_montgomery_blinded_ct( product_share.value().add_mod_ct(&product_blind.value(), modulus.value) } -#[allow(clippy::indexing_slicing)] fn montgomery_mul(lhs: Uint, rhs: Uint, modulus: &'static Modulus) -> Uint { debug_assert!(L <= 6); @@ -4015,27 +4033,32 @@ fn montgomery_mul(lhs: Uint, rhs: Uint, modulus: &'static for i in 0..L { let mut carry = 0u64; for j in 0..L { - let k = i + j; + let k = i.strict_add(j); (limbs[k], carry) = mac_limb(limbs[k], lhs.0[i], rhs.0[j], carry); } - add_limb(&mut limbs, i + L, carry); + add_limb(&mut limbs, i.strict_add(L), carry); } for i in 0..L { let factor = mul_u64_low(limbs[i], modulus.n0_inv); let mut carry = 0u64; for j in 0..L { - let k = i + j; + let k = i.strict_add(j); (limbs[k], carry) = mac_limb(limbs[k], factor, modulus.value.0[j], carry); } - add_limb(&mut limbs, i + L, carry); + add_limb(&mut limbs, i.strict_add(L), carry); } + let double_limbs = L.strict_mul(2); let mut out = [0u64; L]; - out.copy_from_slice(&limbs[L..L + L]); + out.copy_from_slice(&limbs[L..double_limbs]); let out = Uint(out); let (reduced, borrow) = out.sub_raw(&modulus.value); - Uint::select(out, reduced, mask_nonzero_u64(limbs[L + L]) | mask_zero_u64(borrow)) + Uint::select( + out, + reduced, + mask_nonzero_u64(limbs[double_limbs]) | mask_zero_u64(borrow), + ) } fn montgomery_square(value: Uint, modulus: &'static Modulus) -> Uint { @@ -4095,7 +4118,7 @@ fn montgomery_mul_p256_order(lhs: [u64; 4], rhs: [u64; 4]) -> Uint<4> { t1 = v1; t2 = v2; t3 = v3; - t4 = carry_extra + carry; + t4 = carry_extra.strict_add(carry); }}; } @@ -4159,7 +4182,7 @@ fn montgomery_mul_p384_order(lhs: [u64; 6], rhs: [u64; 6]) -> Uint<6> { t3 = v3; t4 = v4; t5 = v5; - t6 = carry_extra + carry; + t6 = carry_extra.strict_add(carry); }}; } @@ -4260,7 +4283,6 @@ fn montgomery_mul_p256_field(lhs: [u64; 4], rhs: [u64; 4]) -> Uint<4> { montgomery_reduce_p256_field([w0, w1, w2, w3, w4, w5, w6, w7]) } -#[allow(clippy::indexing_slicing)] fn montgomery_reduce_p256_field(limbs: [u64; 8]) -> Uint<4> { let [r0, r1, r2, r3, r4, r5, r6, r7] = limbs; let p = P256_FIELD.0; @@ -4306,8 +4328,9 @@ fn sub_p256_field_once(limbs: [u64; 5]) -> Uint<4> { #[inline(always)] fn adc_limb(lhs: u64, rhs: u64, carry: u64) -> (u64, u64) { - let result = u128::from(lhs) + u128::from(rhs) + u128::from(carry); - (result as u64, (result >> 64) as u64) + let (result, overflow0) = lhs.overflowing_add(rhs); + let (result, overflow1) = result.overflowing_add(carry); + (result, u64::from(overflow0 | overflow1)) } #[inline(always)] @@ -4319,29 +4342,32 @@ fn sbb_limb(lhs: u64, rhs: u64, borrow: u64) -> (u64, u64) { #[inline(always)] fn mac_limb(acc: u64, lhs: u64, rhs: u64, carry: u64) -> (u64, u64) { - #[cfg(any(test, target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x"))] - { - let (product_lo, product_hi) = mul_u64_wide(lhs, rhs); - let (result, carry0) = product_lo.overflowing_add(acc); - let (result, carry1) = result.overflowing_add(carry); - let (high, overflow0) = product_hi.overflowing_add(u64::from(carry0)); - let (high, overflow1) = high.overflowing_add(u64::from(carry1)); + let (product_lo, product_hi) = mul_u64_wide(lhs, rhs); + let (result, carry0) = product_lo.overflowing_add(acc); + let (result, carry1) = result.overflowing_add(carry); + let (high, overflow0) = product_hi.overflowing_add(u64::from(carry0)); + let (high, overflow1) = high.overflowing_add(u64::from(carry1)); - // lhs*rhs + acc + carry is at most 2^128 - 1, so the high limb cannot overflow. - // Keep that invariant checked in debug builds without emitting secret-fed panic branches - // in release ECDSA arithmetic on s390x and RISC-V. - debug_assert!(!overflow0 && !overflow1); - (result, high) - } + // lhs*rhs + acc + carry is at most 2^128 - 1, so the high limb cannot overflow. + // Keep that invariant checked in debug builds without emitting secret-fed panic branches + // in release ECDSA arithmetic. + debug_assert!(!overflow0 && !overflow1); + (result, high) +} - #[cfg(not(any(test, target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x")))] - { - let result = u128::from(acc) + (u128::from(lhs) * u128::from(rhs)) + u128::from(carry); - (result as u64, (result >> 64) as u64) - } +#[inline(always)] +#[cfg(any( + test, + not(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x")) +))] +fn split_u128(value: u128) -> (u64, u64) { + let [b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15] = value.to_le_bytes(); + ( + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]), + u64::from_le_bytes([b8, b9, b10, b11, b12, b13, b14, b15]), + ) } -#[allow(clippy::indexing_slicing)] fn add_limb(limbs: &mut [u64; 13], mut index: usize, mut value: u64) { while index < limbs.len() { let (sum, carry) = limbs[index].overflowing_add(value); @@ -4461,7 +4487,8 @@ mod tests { assert_eq!(reader.read_len(), Err(EcdsaError::MalformedDer)); } - let oversized_len_len = [0x80 | (core::mem::size_of::() as u8 + 1)]; + let length_bytes = u8::try_from(core::mem::size_of::()).expect("usize width fits in one DER length byte"); + let oversized_len_len = [0x80 | length_bytes.strict_add(1)]; let mut reader = DerReader::new(&oversized_len_len); assert_eq!(reader.read_len(), Err(EcdsaError::MalformedDer)); } @@ -4471,7 +4498,7 @@ mod tests { sec1[0] = 0x04; P256_GX.write_be(&mut sec1[1..33]); P256_GY.write_be(&mut sec1[33..]); - EcdsaP256PublicKey::from_sec1_bytes(&sec1).unwrap() + EcdsaP256PublicKey::from_sec1_bytes(&sec1).expect("the P-256 generator is a valid public key") } fn p384_public_key() -> EcdsaP384PublicKey { @@ -4479,7 +4506,7 @@ mod tests { sec1[0] = 0x04; P384_GX.write_be(&mut sec1[1..49]); P384_GY.write_be(&mut sec1[49..]); - EcdsaP384PublicKey::from_sec1_bytes(&sec1).unwrap() + EcdsaP384PublicKey::from_sec1_bytes(&sec1).expect("the P-384 generator is a valid public key") } fn p384_sparse_scalar(bits: &[usize]) -> Uint<6> { @@ -4504,12 +4531,12 @@ mod tests { #[test] fn ct_mul_u64_wide_matches_u128_for_edges_and_generated_inputs() { - const EDGES: [u64; 8] = [0, 1, 2, u32::MAX as u64, 1u64 << 32, 1u64 << 63, u64::MAX - 1, u64::MAX]; + const EDGES: [u64; 8] = [0, 1, 2, 0xffff_ffff, 1u64 << 32, 1u64 << 63, u64::MAX - 1, u64::MAX]; for lhs in EDGES { for rhs in EDGES { - let product = u128::from(lhs) * u128::from(rhs); - assert_eq!(ct_mul_u64_wide(lhs, rhs), (product as u64, (product >> 64) as u64)); + let product = u128::from(lhs).strict_mul(u128::from(rhs)); + assert_eq!(ct_mul_u64_wide(lhs, rhs), split_u128(product)); } } @@ -4518,24 +4545,24 @@ mod tests { for _ in 0..1024 { lhs = lhs.wrapping_mul(0x9e37_79b9_7f4a_7c15).wrapping_add(1); rhs = rhs.wrapping_mul(0xd134_2543_de82_ef95).wrapping_add(1); - let product = u128::from(lhs) * u128::from(rhs); - assert_eq!(ct_mul_u64_wide(lhs, rhs), (product as u64, (product >> 64) as u64)); + let product = u128::from(lhs).strict_mul(u128::from(rhs)); + assert_eq!(ct_mul_u64_wide(lhs, rhs), split_u128(product)); } } #[test] fn mac_limb_matches_u128_for_edges_and_generated_inputs() { - const EDGES: [u64; 8] = [0, 1, 2, u32::MAX as u64, 1u64 << 32, 1u64 << 63, u64::MAX - 1, u64::MAX]; + const EDGES: [u64; 8] = [0, 1, 2, 0xffff_ffff, 1u64 << 32, 1u64 << 63, u64::MAX - 1, u64::MAX]; for acc in EDGES { for lhs in EDGES { for rhs in EDGES { for carry in EDGES { - let expected = u128::from(lhs) * u128::from(rhs) + u128::from(acc) + u128::from(carry); - assert_eq!( - mac_limb(acc, lhs, rhs, carry), - (expected as u64, (expected >> 64) as u64) - ); + let expected = u128::from(lhs) + .strict_mul(u128::from(rhs)) + .strict_add(u128::from(acc)) + .strict_add(u128::from(carry)); + assert_eq!(mac_limb(acc, lhs, rhs, carry), split_u128(expected)); } } } @@ -4550,11 +4577,11 @@ mod tests { let rhs = state; state = state.wrapping_mul(0xa409_3822_299f_31d0).wrapping_add(1); let carry = state; - let expected = u128::from(lhs) * u128::from(rhs) + u128::from(acc) + u128::from(carry); - assert_eq!( - mac_limb(acc, lhs, rhs, carry), - (expected as u64, (expected >> 64) as u64) - ); + let expected = u128::from(lhs) + .strict_mul(u128::from(rhs)) + .strict_add(u128::from(acc)) + .strict_add(u128::from(carry)); + assert_eq!(mac_limb(acc, lhs, rhs, carry), split_u128(expected)); } } @@ -4612,7 +4639,7 @@ mod tests { ); bytes[63] = 1; - assert!(EcdsaP256Signature::from_bytes(bytes).is_ok()); + EcdsaP256Signature::from_bytes(bytes).expect("nonzero in-range P-256 scalars form a valid signature encoding"); let mut out_of_range = bytes; P256_ORDER.write_be(&mut out_of_range[..32]); @@ -4632,7 +4659,7 @@ mod tests { ); bytes[95] = 1; - assert!(EcdsaP384Signature::from_bytes(bytes).is_ok()); + EcdsaP384Signature::from_bytes(bytes).expect("nonzero in-range P-384 scalars form a valid signature encoding"); let mut out_of_range = bytes; P384_ORDER.write_be(&mut out_of_range[..48]); @@ -4645,7 +4672,7 @@ mod tests { #[test] fn der_signature_parser_requires_canonical_unsigned_integers() { let good = [0x30, 0x06, 0x02, 0x01, 0x01, 0x02, 0x01, 0x01]; - assert!(EcdsaP256Signature::from_der(&good).is_ok()); + EcdsaP256Signature::from_der(&good).expect("canonical unsigned DER integers must parse"); let negative = [0x30, 0x07, 0x02, 0x02, 0x80, 0x01, 0x02, 0x01, 0x01]; assert_eq!( @@ -4693,8 +4720,8 @@ mod tests { #[test] fn p256_secret_key_one_derives_generator_public_key() { let mut one = [0u8; EcdsaP256SecretKey::LENGTH]; - one[EcdsaP256SecretKey::LENGTH - 1] = 1; - let secret = EcdsaP256SecretKey::from_bytes(one).unwrap(); + *one.last_mut().expect("P-256 scalar encoding is nonempty") = 1; + let secret = EcdsaP256SecretKey::from_bytes(one).expect("one is a valid P-256 secret scalar"); assert_eq!(secret.public_key(), p256_public_key()); assert_eq!(secret.public_key_blinded(|blind| blind.fill(0xa5)), p256_public_key()); @@ -4703,8 +4730,8 @@ mod tests { #[test] fn p384_secret_key_one_derives_generator_public_key() { let mut one = [0u8; EcdsaP384SecretKey::LENGTH]; - one[EcdsaP384SecretKey::LENGTH - 1] = 1; - let secret = EcdsaP384SecretKey::from_bytes(one).unwrap(); + *one.last_mut().expect("P-384 scalar encoding is nonempty") = 1; + let secret = EcdsaP384SecretKey::from_bytes(one).expect("one is a valid P-384 secret scalar"); assert_eq!(secret.public_key(), p384_public_key()); assert_eq!(secret.public_key_blinded(|blind| blind.fill(0x5a)), p384_public_key()); @@ -4712,21 +4739,28 @@ mod tests { #[test] fn p256_try_sign_is_deterministic_low_s_and_verifies() { - let secret = EcdsaP256SecretKey::from_bytes([0x42; EcdsaP256SecretKey::LENGTH]).unwrap(); + let secret = EcdsaP256SecretKey::from_bytes([0x42; EcdsaP256SecretKey::LENGTH]) + .expect("fixture is a valid P-256 secret scalar"); let public = secret.public_key(); let message = b"rscrypto ecdsa p256 signing"; - let first = secret.try_sign(message).unwrap(); - let second = secret.try_sign(message).unwrap(); - let blinded = secret.try_sign_blinded(message, |blind| blind.fill(0x7b)).unwrap(); + let first = secret.try_sign(message).expect("valid P-256 signing fixture must sign"); + let second = secret.try_sign(message).expect("valid P-256 signing fixture must sign"); + let blinded = secret + .try_sign_blinded(message, |blind| blind.fill(0x7b)) + .expect("valid P-256 blinded signing fixture must sign"); assert_eq!(first, second); - assert!(public.verify(message, &first).is_ok()); - assert!(public.verify(message, &blinded).is_ok()); + public + .verify(message, &first) + .expect("fresh P-256 signature must verify"); + public + .verify(message, &blinded) + .expect("fresh blinded P-256 signature must verify"); assert_eq!(first, blinded); assert!( Uint::from_be_slice(&first.as_bytes()[32..]) - .unwrap() + .expect("P-256 signature scalar has the fixed field width") .cmp(&P256_ORDER_HALF) .is_le() ); @@ -4734,21 +4768,28 @@ mod tests { #[test] fn p384_try_sign_is_deterministic_low_s_and_verifies() { - let secret = EcdsaP384SecretKey::from_bytes([0x24; EcdsaP384SecretKey::LENGTH]).unwrap(); + let secret = EcdsaP384SecretKey::from_bytes([0x24; EcdsaP384SecretKey::LENGTH]) + .expect("fixture is a valid P-384 secret scalar"); let public = secret.public_key(); let message = b"rscrypto ecdsa p384 signing"; - let first = secret.try_sign(message).unwrap(); - let second = secret.try_sign(message).unwrap(); - let blinded = secret.try_sign_blinded(message, |blind| blind.fill(0xb7)).unwrap(); + let first = secret.try_sign(message).expect("valid P-384 signing fixture must sign"); + let second = secret.try_sign(message).expect("valid P-384 signing fixture must sign"); + let blinded = secret + .try_sign_blinded(message, |blind| blind.fill(0xb7)) + .expect("valid P-384 blinded signing fixture must sign"); assert_eq!(first, second); - assert!(public.verify(message, &first).is_ok()); - assert!(public.verify(message, &blinded).is_ok()); + public + .verify(message, &first) + .expect("fresh P-384 signature must verify"); + public + .verify(message, &blinded) + .expect("fresh blinded P-384 signature must verify"); assert_eq!(first, blinded); assert!( Uint::from_be_slice(&first.as_bytes()[48..]) - .unwrap() + .expect("P-384 signature scalar has the fixed field width") .cmp(&P384_ORDER_HALF) .is_le() ); @@ -4809,6 +4850,36 @@ mod tests { } } + #[cfg(any( + all(target_arch = "aarch64", any(target_os = "macos", target_os = "linux")), + all(target_arch = "x86_64", target_os = "linux") + ))] + #[test] + fn p384_portable_field_arithmetic_matches_platform_backend() { + let values = [ + Uint::ZERO, + P384_FIELD_MONTGOMERY_ONE, + P384_GX, + P384_GY, + P384_B, + P384_FIELD.sub_raw(&Uint::ONE).0, + ]; + + for lhs in values { + assert_eq!( + ecdsa_p384_field::square(lhs.0), + ecdsa_platform_asm::p384_field_square(&lhs.0) + ); + + for rhs in values { + assert_eq!( + ecdsa_p384_field::mul(lhs.0, rhs.0), + ecdsa_platform_asm::p384_field_mul(&lhs.0, &rhs.0) + ); + } + } + } + #[cfg(any( all(target_arch = "aarch64", any(target_os = "macos", target_os = "linux")), all(target_arch = "x86_64", target_os = "linux") @@ -4852,8 +4923,8 @@ mod tests { for row in [0usize, 1, 2, 23, 46] { for column in [0usize, 1, 3, 7] { - let bit = row + column * P384_SIGNING_COMB_ROWS; - let next_bit = bit + 1; + let bit = row.strict_add(column.strict_mul(P384_SIGNING_COMB_ROWS)); + let next_bit = bit.strict_add(1); assert_p384_nonexceptional_comb_matches_complete(p384_sparse_scalar(&[bit, next_bit])); } } @@ -4888,8 +4959,10 @@ mod tests { fn p384_owned_wide_order_reduction_matches_platform_reduction() { for bytes in [[0u8; 96], [0xffu8; 96], { let mut bytes = [0u8; 96]; - for (index, byte) in bytes.iter_mut().enumerate() { - *byte = (index as u8).wrapping_mul(17).wrapping_add(0xa5); + let mut value = 0xa5u8; + for byte in &mut bytes { + *byte = value; + value = value.wrapping_add(17); } bytes }] { @@ -4902,8 +4975,10 @@ mod tests { #[test] fn ecdsa_secret_key_debug_is_redacted() { - let p256 = EcdsaP256SecretKey::from_bytes([0x11; EcdsaP256SecretKey::LENGTH]).unwrap(); - let p384 = EcdsaP384SecretKey::from_bytes([0x22; EcdsaP384SecretKey::LENGTH]).unwrap(); + let p256 = EcdsaP256SecretKey::from_bytes([0x11; EcdsaP256SecretKey::LENGTH]) + .expect("fixture is a valid P-256 secret scalar"); + let p384 = EcdsaP384SecretKey::from_bytes([0x22; EcdsaP384SecretKey::LENGTH]) + .expect("fixture is a valid P-384 secret scalar"); assert_eq!(format!("{p256:?}"), "EcdsaP256SecretKey(****)"); assert_eq!(format!("{p384:?}"), "EcdsaP384SecretKey(****)"); diff --git a/src/auth/ecdsa_aarch64_asm.rs b/src/auth/ecdsa_aarch64_asm.rs index 5d515d6e..cce0b20d 100644 --- a/src/auth/ecdsa_aarch64_asm.rs +++ b/src/auth/ecdsa_aarch64_asm.rs @@ -5,8 +5,6 @@ //! ABI boundary; `ecdsa.rs` owns scalar validation, blinding, affine //! conversion, and public API semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; use super::ZeroizingWords; diff --git a/src/auth/ecdsa_p384_field.rs b/src/auth/ecdsa_p384_field.rs index 64c91d30..65f544c0 100644 --- a/src/auth/ecdsa_p384_field.rs +++ b/src/auth/ecdsa_p384_field.rs @@ -1,49 +1,23 @@ -//! Fiat-Crypto P-384 field arithmetic used by ECDSA verification. +//! Fiat-Crypto P-384 field arithmetic used by portable ECDSA paths. //! -//! This file contains a narrow extraction of generated Fiat-Crypto code +//! This file contains a narrow, adapted extraction of generated Fiat-Crypto code //! from the RustCrypto `p384` crate field backend. The generated source //! is distributed under MIT, Apache-2.0, or BSD-1-Clause terms by the //! Fiat-Crypto authors. It is kept private to this module and wrapped by //! small `mul`/`square` helpers below. -#![allow( - non_camel_case_types, - clippy::identity_op, - clippy::too_many_arguments, - clippy::unnecessary_cast, - clippy::unused_unit, - unused_assignments, - unused_variables, - dead_code, - unused_parens -)] - -type fiat_p384_u1 = u8; -type fiat_p384_i1 = i8; -type fiat_p384_u2 = u8; -type fiat_p384_i2 = i8; -type fiat_p384_montgomery_domain_field_element = [u64; 6]; - -const fn fiat_p384_addcarryx_u64(arg1: fiat_p384_u1, arg2: u64, arg3: u64) -> (u64, fiat_p384_u1) { - let mut out1: u64 = 0; - let mut out2: fiat_p384_u1 = 0; - let x1: u128 = (((arg1 as u128) + (arg2 as u128)) + (arg3 as u128)); - let x2: u64 = ((x1 & (0xffffffffffffffff as u128)) as u64); - let x3: fiat_p384_u1 = ((x1 >> 64) as fiat_p384_u1); - out1 = x2; - out2 = x3; - (out1, out2) +#[inline(always)] +fn fiat_p384_addcarryx_u64(carry: u8, lhs: u64, rhs: u64) -> (u64, u8) { + let (sum, overflow0) = lhs.overflowing_add(rhs); + let (sum, overflow1) = sum.overflowing_add(u64::from(carry)); + (sum, u8::from(overflow0 | overflow1)) } -const fn fiat_p384_subborrowx_u64(arg1: fiat_p384_u1, arg2: u64, arg3: u64) -> (u64, fiat_p384_u1) { - let mut out1: u64 = 0; - let mut out2: fiat_p384_u1 = 0; - let x1: i128 = (((arg2 as i128) - (arg1 as i128)) - (arg3 as i128)); - let x2: fiat_p384_i1 = ((x1 >> 64) as fiat_p384_i1); - let x3: u64 = ((x1 & (0xffffffffffffffff as i128)) as u64); - out1 = x3; - out2 = (((0x0 as fiat_p384_i2) - (x2 as fiat_p384_i2)) as fiat_p384_u1); - (out1, out2) +#[inline(always)] +fn fiat_p384_subborrowx_u64(borrow: u8, lhs: u64, rhs: u64) -> (u64, u8) { + let (difference, underflow0) = lhs.overflowing_sub(rhs); + let (difference, underflow1) = difference.overflowing_sub(u64::from(borrow)); + (difference, u8::from(underflow0 | underflow1)) } #[inline(always)] @@ -51,40 +25,33 @@ fn fiat_p384_mulx_u64(arg1: u64, arg2: u64) -> (u64, u64) { super::mul_u64_wide(arg1, arg2) } -const fn fiat_p384_cmovznz_u64(arg1: fiat_p384_u1, arg2: u64, arg3: u64) -> u64 { - let mut out1: u64 = 0; - let x1: fiat_p384_u1 = (!(!arg1)); - let x2: u64 = ((((((0x0 as fiat_p384_i2) - (x1 as fiat_p384_i2)) as fiat_p384_i1) as i128) - & (0xffffffffffffffff as i128)) as u64); - let x3: u64 = ((x2 & arg3) | ((!x2) & arg2)); - out1 = x3; - out1 +#[inline(always)] +fn fiat_p384_cmovznz_u64(choice: u8, zero: u64, nonzero: u64) -> u64 { + let mask = u64::from(choice).wrapping_neg(); + (mask & nonzero) | (!mask & zero) } -fn fiat_p384_mul( - arg1: &fiat_p384_montgomery_domain_field_element, - arg2: &fiat_p384_montgomery_domain_field_element, -) -> fiat_p384_montgomery_domain_field_element { - let mut out1: fiat_p384_montgomery_domain_field_element = [0; 6]; - let x1: u64 = (arg1[1]); - let x2: u64 = (arg1[2]); - let x3: u64 = (arg1[3]); - let x4: u64 = (arg1[4]); - let x5: u64 = (arg1[5]); - let x6: u64 = (arg1[0]); - let (x7, x8) = fiat_p384_mulx_u64(x6, (arg2[5])); - let (x9, x10) = fiat_p384_mulx_u64(x6, (arg2[4])); - let (x11, x12) = fiat_p384_mulx_u64(x6, (arg2[3])); - let (x13, x14) = fiat_p384_mulx_u64(x6, (arg2[2])); - let (x15, x16) = fiat_p384_mulx_u64(x6, (arg2[1])); - let (x17, x18) = fiat_p384_mulx_u64(x6, (arg2[0])); +fn fiat_p384_mul(arg1: &[u64; 6], arg2: &[u64; 6]) -> [u64; 6] { + let mut out1 = [0; 6]; + let x1: u64 = arg1[1]; + let x2: u64 = arg1[2]; + let x3: u64 = arg1[3]; + let x4: u64 = arg1[4]; + let x5: u64 = arg1[5]; + let x6: u64 = arg1[0]; + let (x7, x8) = fiat_p384_mulx_u64(x6, arg2[5]); + let (x9, x10) = fiat_p384_mulx_u64(x6, arg2[4]); + let (x11, x12) = fiat_p384_mulx_u64(x6, arg2[3]); + let (x13, x14) = fiat_p384_mulx_u64(x6, arg2[2]); + let (x15, x16) = fiat_p384_mulx_u64(x6, arg2[1]); + let (x17, x18) = fiat_p384_mulx_u64(x6, arg2[0]); let (x19, x20) = fiat_p384_addcarryx_u64(0x0, x18, x15); let (x21, x22) = fiat_p384_addcarryx_u64(x20, x16, x13); let (x23, x24) = fiat_p384_addcarryx_u64(x22, x14, x11); let (x25, x26) = fiat_p384_addcarryx_u64(x24, x12, x9); let (x27, x28) = fiat_p384_addcarryx_u64(x26, x10, x7); - let x29: u64 = ((x28 as u64) + x8); - let (x30, x31) = fiat_p384_mulx_u64(x17, 0x100000001); + let x29: u64 = u64::from(x28).wrapping_add(x8); + let (x30, _x31) = fiat_p384_mulx_u64(x17, 0x100000001); let (x32, x33) = fiat_p384_mulx_u64(x30, 0xffffffffffffffff); let (x34, x35) = fiat_p384_mulx_u64(x30, 0xffffffffffffffff); let (x36, x37) = fiat_p384_mulx_u64(x30, 0xffffffffffffffff); @@ -96,34 +63,34 @@ fn fiat_p384_mul( let (x48, x49) = fiat_p384_addcarryx_u64(x47, x39, x36); let (x50, x51) = fiat_p384_addcarryx_u64(x49, x37, x34); let (x52, x53) = fiat_p384_addcarryx_u64(x51, x35, x32); - let x54: u64 = ((x53 as u64) + x33); - let (x55, x56) = fiat_p384_addcarryx_u64(0x0, x17, x42); + let x54: u64 = u64::from(x53).wrapping_add(x33); + let (_x55, x56) = fiat_p384_addcarryx_u64(0x0, x17, x42); let (x57, x58) = fiat_p384_addcarryx_u64(x56, x19, x44); let (x59, x60) = fiat_p384_addcarryx_u64(x58, x21, x46); let (x61, x62) = fiat_p384_addcarryx_u64(x60, x23, x48); let (x63, x64) = fiat_p384_addcarryx_u64(x62, x25, x50); let (x65, x66) = fiat_p384_addcarryx_u64(x64, x27, x52); let (x67, x68) = fiat_p384_addcarryx_u64(x66, x29, x54); - let (x69, x70) = fiat_p384_mulx_u64(x1, (arg2[5])); - let (x71, x72) = fiat_p384_mulx_u64(x1, (arg2[4])); - let (x73, x74) = fiat_p384_mulx_u64(x1, (arg2[3])); - let (x75, x76) = fiat_p384_mulx_u64(x1, (arg2[2])); - let (x77, x78) = fiat_p384_mulx_u64(x1, (arg2[1])); - let (x79, x80) = fiat_p384_mulx_u64(x1, (arg2[0])); + let (x69, x70) = fiat_p384_mulx_u64(x1, arg2[5]); + let (x71, x72) = fiat_p384_mulx_u64(x1, arg2[4]); + let (x73, x74) = fiat_p384_mulx_u64(x1, arg2[3]); + let (x75, x76) = fiat_p384_mulx_u64(x1, arg2[2]); + let (x77, x78) = fiat_p384_mulx_u64(x1, arg2[1]); + let (x79, x80) = fiat_p384_mulx_u64(x1, arg2[0]); let (x81, x82) = fiat_p384_addcarryx_u64(0x0, x80, x77); let (x83, x84) = fiat_p384_addcarryx_u64(x82, x78, x75); let (x85, x86) = fiat_p384_addcarryx_u64(x84, x76, x73); let (x87, x88) = fiat_p384_addcarryx_u64(x86, x74, x71); let (x89, x90) = fiat_p384_addcarryx_u64(x88, x72, x69); - let x91: u64 = ((x90 as u64) + x70); + let x91: u64 = u64::from(x90).wrapping_add(x70); let (x92, x93) = fiat_p384_addcarryx_u64(0x0, x57, x79); let (x94, x95) = fiat_p384_addcarryx_u64(x93, x59, x81); let (x96, x97) = fiat_p384_addcarryx_u64(x95, x61, x83); let (x98, x99) = fiat_p384_addcarryx_u64(x97, x63, x85); let (x100, x101) = fiat_p384_addcarryx_u64(x99, x65, x87); let (x102, x103) = fiat_p384_addcarryx_u64(x101, x67, x89); - let (x104, x105) = fiat_p384_addcarryx_u64(x103, (x68 as u64), x91); - let (x106, x107) = fiat_p384_mulx_u64(x92, 0x100000001); + let (x104, x105) = fiat_p384_addcarryx_u64(x103, u64::from(x68), x91); + let (x106, _x107) = fiat_p384_mulx_u64(x92, 0x100000001); let (x108, x109) = fiat_p384_mulx_u64(x106, 0xffffffffffffffff); let (x110, x111) = fiat_p384_mulx_u64(x106, 0xffffffffffffffff); let (x112, x113) = fiat_p384_mulx_u64(x106, 0xffffffffffffffff); @@ -135,27 +102,27 @@ fn fiat_p384_mul( let (x124, x125) = fiat_p384_addcarryx_u64(x123, x115, x112); let (x126, x127) = fiat_p384_addcarryx_u64(x125, x113, x110); let (x128, x129) = fiat_p384_addcarryx_u64(x127, x111, x108); - let x130: u64 = ((x129 as u64) + x109); - let (x131, x132) = fiat_p384_addcarryx_u64(0x0, x92, x118); + let x130: u64 = u64::from(x129).wrapping_add(x109); + let (_x131, x132) = fiat_p384_addcarryx_u64(0x0, x92, x118); let (x133, x134) = fiat_p384_addcarryx_u64(x132, x94, x120); let (x135, x136) = fiat_p384_addcarryx_u64(x134, x96, x122); let (x137, x138) = fiat_p384_addcarryx_u64(x136, x98, x124); let (x139, x140) = fiat_p384_addcarryx_u64(x138, x100, x126); let (x141, x142) = fiat_p384_addcarryx_u64(x140, x102, x128); let (x143, x144) = fiat_p384_addcarryx_u64(x142, x104, x130); - let x145: u64 = ((x144 as u64) + (x105 as u64)); - let (x146, x147) = fiat_p384_mulx_u64(x2, (arg2[5])); - let (x148, x149) = fiat_p384_mulx_u64(x2, (arg2[4])); - let (x150, x151) = fiat_p384_mulx_u64(x2, (arg2[3])); - let (x152, x153) = fiat_p384_mulx_u64(x2, (arg2[2])); - let (x154, x155) = fiat_p384_mulx_u64(x2, (arg2[1])); - let (x156, x157) = fiat_p384_mulx_u64(x2, (arg2[0])); + let x145: u64 = u64::from(x144).wrapping_add(u64::from(x105)); + let (x146, x147) = fiat_p384_mulx_u64(x2, arg2[5]); + let (x148, x149) = fiat_p384_mulx_u64(x2, arg2[4]); + let (x150, x151) = fiat_p384_mulx_u64(x2, arg2[3]); + let (x152, x153) = fiat_p384_mulx_u64(x2, arg2[2]); + let (x154, x155) = fiat_p384_mulx_u64(x2, arg2[1]); + let (x156, x157) = fiat_p384_mulx_u64(x2, arg2[0]); let (x158, x159) = fiat_p384_addcarryx_u64(0x0, x157, x154); let (x160, x161) = fiat_p384_addcarryx_u64(x159, x155, x152); let (x162, x163) = fiat_p384_addcarryx_u64(x161, x153, x150); let (x164, x165) = fiat_p384_addcarryx_u64(x163, x151, x148); let (x166, x167) = fiat_p384_addcarryx_u64(x165, x149, x146); - let x168: u64 = ((x167 as u64) + x147); + let x168: u64 = u64::from(x167).wrapping_add(x147); let (x169, x170) = fiat_p384_addcarryx_u64(0x0, x133, x156); let (x171, x172) = fiat_p384_addcarryx_u64(x170, x135, x158); let (x173, x174) = fiat_p384_addcarryx_u64(x172, x137, x160); @@ -163,7 +130,7 @@ fn fiat_p384_mul( let (x177, x178) = fiat_p384_addcarryx_u64(x176, x141, x164); let (x179, x180) = fiat_p384_addcarryx_u64(x178, x143, x166); let (x181, x182) = fiat_p384_addcarryx_u64(x180, x145, x168); - let (x183, x184) = fiat_p384_mulx_u64(x169, 0x100000001); + let (x183, _x184) = fiat_p384_mulx_u64(x169, 0x100000001); let (x185, x186) = fiat_p384_mulx_u64(x183, 0xffffffffffffffff); let (x187, x188) = fiat_p384_mulx_u64(x183, 0xffffffffffffffff); let (x189, x190) = fiat_p384_mulx_u64(x183, 0xffffffffffffffff); @@ -175,27 +142,27 @@ fn fiat_p384_mul( let (x201, x202) = fiat_p384_addcarryx_u64(x200, x192, x189); let (x203, x204) = fiat_p384_addcarryx_u64(x202, x190, x187); let (x205, x206) = fiat_p384_addcarryx_u64(x204, x188, x185); - let x207: u64 = ((x206 as u64) + x186); - let (x208, x209) = fiat_p384_addcarryx_u64(0x0, x169, x195); + let x207: u64 = u64::from(x206).wrapping_add(x186); + let (_x208, x209) = fiat_p384_addcarryx_u64(0x0, x169, x195); let (x210, x211) = fiat_p384_addcarryx_u64(x209, x171, x197); let (x212, x213) = fiat_p384_addcarryx_u64(x211, x173, x199); let (x214, x215) = fiat_p384_addcarryx_u64(x213, x175, x201); let (x216, x217) = fiat_p384_addcarryx_u64(x215, x177, x203); let (x218, x219) = fiat_p384_addcarryx_u64(x217, x179, x205); let (x220, x221) = fiat_p384_addcarryx_u64(x219, x181, x207); - let x222: u64 = ((x221 as u64) + (x182 as u64)); - let (x223, x224) = fiat_p384_mulx_u64(x3, (arg2[5])); - let (x225, x226) = fiat_p384_mulx_u64(x3, (arg2[4])); - let (x227, x228) = fiat_p384_mulx_u64(x3, (arg2[3])); - let (x229, x230) = fiat_p384_mulx_u64(x3, (arg2[2])); - let (x231, x232) = fiat_p384_mulx_u64(x3, (arg2[1])); - let (x233, x234) = fiat_p384_mulx_u64(x3, (arg2[0])); + let x222: u64 = u64::from(x221).wrapping_add(u64::from(x182)); + let (x223, x224) = fiat_p384_mulx_u64(x3, arg2[5]); + let (x225, x226) = fiat_p384_mulx_u64(x3, arg2[4]); + let (x227, x228) = fiat_p384_mulx_u64(x3, arg2[3]); + let (x229, x230) = fiat_p384_mulx_u64(x3, arg2[2]); + let (x231, x232) = fiat_p384_mulx_u64(x3, arg2[1]); + let (x233, x234) = fiat_p384_mulx_u64(x3, arg2[0]); let (x235, x236) = fiat_p384_addcarryx_u64(0x0, x234, x231); let (x237, x238) = fiat_p384_addcarryx_u64(x236, x232, x229); let (x239, x240) = fiat_p384_addcarryx_u64(x238, x230, x227); let (x241, x242) = fiat_p384_addcarryx_u64(x240, x228, x225); let (x243, x244) = fiat_p384_addcarryx_u64(x242, x226, x223); - let x245: u64 = ((x244 as u64) + x224); + let x245: u64 = u64::from(x244).wrapping_add(x224); let (x246, x247) = fiat_p384_addcarryx_u64(0x0, x210, x233); let (x248, x249) = fiat_p384_addcarryx_u64(x247, x212, x235); let (x250, x251) = fiat_p384_addcarryx_u64(x249, x214, x237); @@ -203,7 +170,7 @@ fn fiat_p384_mul( let (x254, x255) = fiat_p384_addcarryx_u64(x253, x218, x241); let (x256, x257) = fiat_p384_addcarryx_u64(x255, x220, x243); let (x258, x259) = fiat_p384_addcarryx_u64(x257, x222, x245); - let (x260, x261) = fiat_p384_mulx_u64(x246, 0x100000001); + let (x260, _x261) = fiat_p384_mulx_u64(x246, 0x100000001); let (x262, x263) = fiat_p384_mulx_u64(x260, 0xffffffffffffffff); let (x264, x265) = fiat_p384_mulx_u64(x260, 0xffffffffffffffff); let (x266, x267) = fiat_p384_mulx_u64(x260, 0xffffffffffffffff); @@ -215,27 +182,27 @@ fn fiat_p384_mul( let (x278, x279) = fiat_p384_addcarryx_u64(x277, x269, x266); let (x280, x281) = fiat_p384_addcarryx_u64(x279, x267, x264); let (x282, x283) = fiat_p384_addcarryx_u64(x281, x265, x262); - let x284: u64 = ((x283 as u64) + x263); - let (x285, x286) = fiat_p384_addcarryx_u64(0x0, x246, x272); + let x284: u64 = u64::from(x283).wrapping_add(x263); + let (_x285, x286) = fiat_p384_addcarryx_u64(0x0, x246, x272); let (x287, x288) = fiat_p384_addcarryx_u64(x286, x248, x274); let (x289, x290) = fiat_p384_addcarryx_u64(x288, x250, x276); let (x291, x292) = fiat_p384_addcarryx_u64(x290, x252, x278); let (x293, x294) = fiat_p384_addcarryx_u64(x292, x254, x280); let (x295, x296) = fiat_p384_addcarryx_u64(x294, x256, x282); let (x297, x298) = fiat_p384_addcarryx_u64(x296, x258, x284); - let x299: u64 = ((x298 as u64) + (x259 as u64)); - let (x300, x301) = fiat_p384_mulx_u64(x4, (arg2[5])); - let (x302, x303) = fiat_p384_mulx_u64(x4, (arg2[4])); - let (x304, x305) = fiat_p384_mulx_u64(x4, (arg2[3])); - let (x306, x307) = fiat_p384_mulx_u64(x4, (arg2[2])); - let (x308, x309) = fiat_p384_mulx_u64(x4, (arg2[1])); - let (x310, x311) = fiat_p384_mulx_u64(x4, (arg2[0])); + let x299: u64 = u64::from(x298).wrapping_add(u64::from(x259)); + let (x300, x301) = fiat_p384_mulx_u64(x4, arg2[5]); + let (x302, x303) = fiat_p384_mulx_u64(x4, arg2[4]); + let (x304, x305) = fiat_p384_mulx_u64(x4, arg2[3]); + let (x306, x307) = fiat_p384_mulx_u64(x4, arg2[2]); + let (x308, x309) = fiat_p384_mulx_u64(x4, arg2[1]); + let (x310, x311) = fiat_p384_mulx_u64(x4, arg2[0]); let (x312, x313) = fiat_p384_addcarryx_u64(0x0, x311, x308); let (x314, x315) = fiat_p384_addcarryx_u64(x313, x309, x306); let (x316, x317) = fiat_p384_addcarryx_u64(x315, x307, x304); let (x318, x319) = fiat_p384_addcarryx_u64(x317, x305, x302); let (x320, x321) = fiat_p384_addcarryx_u64(x319, x303, x300); - let x322: u64 = ((x321 as u64) + x301); + let x322: u64 = u64::from(x321).wrapping_add(x301); let (x323, x324) = fiat_p384_addcarryx_u64(0x0, x287, x310); let (x325, x326) = fiat_p384_addcarryx_u64(x324, x289, x312); let (x327, x328) = fiat_p384_addcarryx_u64(x326, x291, x314); @@ -243,7 +210,7 @@ fn fiat_p384_mul( let (x331, x332) = fiat_p384_addcarryx_u64(x330, x295, x318); let (x333, x334) = fiat_p384_addcarryx_u64(x332, x297, x320); let (x335, x336) = fiat_p384_addcarryx_u64(x334, x299, x322); - let (x337, x338) = fiat_p384_mulx_u64(x323, 0x100000001); + let (x337, _x338) = fiat_p384_mulx_u64(x323, 0x100000001); let (x339, x340) = fiat_p384_mulx_u64(x337, 0xffffffffffffffff); let (x341, x342) = fiat_p384_mulx_u64(x337, 0xffffffffffffffff); let (x343, x344) = fiat_p384_mulx_u64(x337, 0xffffffffffffffff); @@ -255,27 +222,27 @@ fn fiat_p384_mul( let (x355, x356) = fiat_p384_addcarryx_u64(x354, x346, x343); let (x357, x358) = fiat_p384_addcarryx_u64(x356, x344, x341); let (x359, x360) = fiat_p384_addcarryx_u64(x358, x342, x339); - let x361: u64 = ((x360 as u64) + x340); - let (x362, x363) = fiat_p384_addcarryx_u64(0x0, x323, x349); + let x361: u64 = u64::from(x360).wrapping_add(x340); + let (_x362, x363) = fiat_p384_addcarryx_u64(0x0, x323, x349); let (x364, x365) = fiat_p384_addcarryx_u64(x363, x325, x351); let (x366, x367) = fiat_p384_addcarryx_u64(x365, x327, x353); let (x368, x369) = fiat_p384_addcarryx_u64(x367, x329, x355); let (x370, x371) = fiat_p384_addcarryx_u64(x369, x331, x357); let (x372, x373) = fiat_p384_addcarryx_u64(x371, x333, x359); let (x374, x375) = fiat_p384_addcarryx_u64(x373, x335, x361); - let x376: u64 = ((x375 as u64) + (x336 as u64)); - let (x377, x378) = fiat_p384_mulx_u64(x5, (arg2[5])); - let (x379, x380) = fiat_p384_mulx_u64(x5, (arg2[4])); - let (x381, x382) = fiat_p384_mulx_u64(x5, (arg2[3])); - let (x383, x384) = fiat_p384_mulx_u64(x5, (arg2[2])); - let (x385, x386) = fiat_p384_mulx_u64(x5, (arg2[1])); - let (x387, x388) = fiat_p384_mulx_u64(x5, (arg2[0])); + let x376: u64 = u64::from(x375).wrapping_add(u64::from(x336)); + let (x377, x378) = fiat_p384_mulx_u64(x5, arg2[5]); + let (x379, x380) = fiat_p384_mulx_u64(x5, arg2[4]); + let (x381, x382) = fiat_p384_mulx_u64(x5, arg2[3]); + let (x383, x384) = fiat_p384_mulx_u64(x5, arg2[2]); + let (x385, x386) = fiat_p384_mulx_u64(x5, arg2[1]); + let (x387, x388) = fiat_p384_mulx_u64(x5, arg2[0]); let (x389, x390) = fiat_p384_addcarryx_u64(0x0, x388, x385); let (x391, x392) = fiat_p384_addcarryx_u64(x390, x386, x383); let (x393, x394) = fiat_p384_addcarryx_u64(x392, x384, x381); let (x395, x396) = fiat_p384_addcarryx_u64(x394, x382, x379); let (x397, x398) = fiat_p384_addcarryx_u64(x396, x380, x377); - let x399: u64 = ((x398 as u64) + x378); + let x399: u64 = u64::from(x398).wrapping_add(x378); let (x400, x401) = fiat_p384_addcarryx_u64(0x0, x364, x387); let (x402, x403) = fiat_p384_addcarryx_u64(x401, x366, x389); let (x404, x405) = fiat_p384_addcarryx_u64(x403, x368, x391); @@ -283,7 +250,7 @@ fn fiat_p384_mul( let (x408, x409) = fiat_p384_addcarryx_u64(x407, x372, x395); let (x410, x411) = fiat_p384_addcarryx_u64(x409, x374, x397); let (x412, x413) = fiat_p384_addcarryx_u64(x411, x376, x399); - let (x414, x415) = fiat_p384_mulx_u64(x400, 0x100000001); + let (x414, _x415) = fiat_p384_mulx_u64(x400, 0x100000001); let (x416, x417) = fiat_p384_mulx_u64(x414, 0xffffffffffffffff); let (x418, x419) = fiat_p384_mulx_u64(x414, 0xffffffffffffffff); let (x420, x421) = fiat_p384_mulx_u64(x414, 0xffffffffffffffff); @@ -295,28 +262,28 @@ fn fiat_p384_mul( let (x432, x433) = fiat_p384_addcarryx_u64(x431, x423, x420); let (x434, x435) = fiat_p384_addcarryx_u64(x433, x421, x418); let (x436, x437) = fiat_p384_addcarryx_u64(x435, x419, x416); - let x438: u64 = ((x437 as u64) + x417); - let (x439, x440) = fiat_p384_addcarryx_u64(0x0, x400, x426); + let x438: u64 = u64::from(x437).wrapping_add(x417); + let (_x439, x440) = fiat_p384_addcarryx_u64(0x0, x400, x426); let (x441, x442) = fiat_p384_addcarryx_u64(x440, x402, x428); let (x443, x444) = fiat_p384_addcarryx_u64(x442, x404, x430); let (x445, x446) = fiat_p384_addcarryx_u64(x444, x406, x432); let (x447, x448) = fiat_p384_addcarryx_u64(x446, x408, x434); let (x449, x450) = fiat_p384_addcarryx_u64(x448, x410, x436); let (x451, x452) = fiat_p384_addcarryx_u64(x450, x412, x438); - let x453: u64 = ((x452 as u64) + (x413 as u64)); + let x453: u64 = u64::from(x452).wrapping_add(u64::from(x413)); let (x454, x455) = fiat_p384_subborrowx_u64(0x0, x441, 0xffffffff); let (x456, x457) = fiat_p384_subborrowx_u64(x455, x443, 0xffffffff00000000); let (x458, x459) = fiat_p384_subborrowx_u64(x457, x445, 0xfffffffffffffffe); let (x460, x461) = fiat_p384_subborrowx_u64(x459, x447, 0xffffffffffffffff); let (x462, x463) = fiat_p384_subborrowx_u64(x461, x449, 0xffffffffffffffff); let (x464, x465) = fiat_p384_subborrowx_u64(x463, x451, 0xffffffffffffffff); - let (x466, x467) = fiat_p384_subborrowx_u64(x465, x453, (0x0 as u64)); - let (x468) = fiat_p384_cmovznz_u64(x467, x454, x441); - let (x469) = fiat_p384_cmovznz_u64(x467, x456, x443); - let (x470) = fiat_p384_cmovznz_u64(x467, x458, x445); - let (x471) = fiat_p384_cmovznz_u64(x467, x460, x447); - let (x472) = fiat_p384_cmovznz_u64(x467, x462, x449); - let (x473) = fiat_p384_cmovznz_u64(x467, x464, x451); + let (_x466, x467) = fiat_p384_subborrowx_u64(x465, x453, 0x0_u64); + let x468 = fiat_p384_cmovznz_u64(x467, x454, x441); + let x469 = fiat_p384_cmovznz_u64(x467, x456, x443); + let x470 = fiat_p384_cmovznz_u64(x467, x458, x445); + let x471 = fiat_p384_cmovznz_u64(x467, x460, x447); + let x472 = fiat_p384_cmovznz_u64(x467, x462, x449); + let x473 = fiat_p384_cmovznz_u64(x467, x464, x451); out1[0] = x468; out1[1] = x469; out1[2] = x470; @@ -326,27 +293,27 @@ fn fiat_p384_mul( out1 } -fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p384_montgomery_domain_field_element { - let mut out1: fiat_p384_montgomery_domain_field_element = [0; 6]; - let x1: u64 = (arg1[1]); - let x2: u64 = (arg1[2]); - let x3: u64 = (arg1[3]); - let x4: u64 = (arg1[4]); - let x5: u64 = (arg1[5]); - let x6: u64 = (arg1[0]); - let (x7, x8) = fiat_p384_mulx_u64(x6, (arg1[5])); - let (x9, x10) = fiat_p384_mulx_u64(x6, (arg1[4])); - let (x11, x12) = fiat_p384_mulx_u64(x6, (arg1[3])); - let (x13, x14) = fiat_p384_mulx_u64(x6, (arg1[2])); - let (x15, x16) = fiat_p384_mulx_u64(x6, (arg1[1])); - let (x17, x18) = fiat_p384_mulx_u64(x6, (arg1[0])); +fn fiat_p384_square(arg1: &[u64; 6]) -> [u64; 6] { + let mut out1 = [0; 6]; + let x1: u64 = arg1[1]; + let x2: u64 = arg1[2]; + let x3: u64 = arg1[3]; + let x4: u64 = arg1[4]; + let x5: u64 = arg1[5]; + let x6: u64 = arg1[0]; + let (x7, x8) = fiat_p384_mulx_u64(x6, arg1[5]); + let (x9, x10) = fiat_p384_mulx_u64(x6, arg1[4]); + let (x11, x12) = fiat_p384_mulx_u64(x6, arg1[3]); + let (x13, x14) = fiat_p384_mulx_u64(x6, arg1[2]); + let (x15, x16) = fiat_p384_mulx_u64(x6, arg1[1]); + let (x17, x18) = fiat_p384_mulx_u64(x6, arg1[0]); let (x19, x20) = fiat_p384_addcarryx_u64(0x0, x18, x15); let (x21, x22) = fiat_p384_addcarryx_u64(x20, x16, x13); let (x23, x24) = fiat_p384_addcarryx_u64(x22, x14, x11); let (x25, x26) = fiat_p384_addcarryx_u64(x24, x12, x9); let (x27, x28) = fiat_p384_addcarryx_u64(x26, x10, x7); - let x29: u64 = ((x28 as u64) + x8); - let (x30, x31) = fiat_p384_mulx_u64(x17, 0x100000001); + let x29: u64 = u64::from(x28).wrapping_add(x8); + let (x30, _x31) = fiat_p384_mulx_u64(x17, 0x100000001); let (x32, x33) = fiat_p384_mulx_u64(x30, 0xffffffffffffffff); let (x34, x35) = fiat_p384_mulx_u64(x30, 0xffffffffffffffff); let (x36, x37) = fiat_p384_mulx_u64(x30, 0xffffffffffffffff); @@ -358,34 +325,34 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x48, x49) = fiat_p384_addcarryx_u64(x47, x39, x36); let (x50, x51) = fiat_p384_addcarryx_u64(x49, x37, x34); let (x52, x53) = fiat_p384_addcarryx_u64(x51, x35, x32); - let x54: u64 = ((x53 as u64) + x33); - let (x55, x56) = fiat_p384_addcarryx_u64(0x0, x17, x42); + let x54: u64 = u64::from(x53).wrapping_add(x33); + let (_x55, x56) = fiat_p384_addcarryx_u64(0x0, x17, x42); let (x57, x58) = fiat_p384_addcarryx_u64(x56, x19, x44); let (x59, x60) = fiat_p384_addcarryx_u64(x58, x21, x46); let (x61, x62) = fiat_p384_addcarryx_u64(x60, x23, x48); let (x63, x64) = fiat_p384_addcarryx_u64(x62, x25, x50); let (x65, x66) = fiat_p384_addcarryx_u64(x64, x27, x52); let (x67, x68) = fiat_p384_addcarryx_u64(x66, x29, x54); - let (x69, x70) = fiat_p384_mulx_u64(x1, (arg1[5])); - let (x71, x72) = fiat_p384_mulx_u64(x1, (arg1[4])); - let (x73, x74) = fiat_p384_mulx_u64(x1, (arg1[3])); - let (x75, x76) = fiat_p384_mulx_u64(x1, (arg1[2])); - let (x77, x78) = fiat_p384_mulx_u64(x1, (arg1[1])); - let (x79, x80) = fiat_p384_mulx_u64(x1, (arg1[0])); + let (x69, x70) = fiat_p384_mulx_u64(x1, arg1[5]); + let (x71, x72) = fiat_p384_mulx_u64(x1, arg1[4]); + let (x73, x74) = fiat_p384_mulx_u64(x1, arg1[3]); + let (x75, x76) = fiat_p384_mulx_u64(x1, arg1[2]); + let (x77, x78) = fiat_p384_mulx_u64(x1, arg1[1]); + let (x79, x80) = fiat_p384_mulx_u64(x1, arg1[0]); let (x81, x82) = fiat_p384_addcarryx_u64(0x0, x80, x77); let (x83, x84) = fiat_p384_addcarryx_u64(x82, x78, x75); let (x85, x86) = fiat_p384_addcarryx_u64(x84, x76, x73); let (x87, x88) = fiat_p384_addcarryx_u64(x86, x74, x71); let (x89, x90) = fiat_p384_addcarryx_u64(x88, x72, x69); - let x91: u64 = ((x90 as u64) + x70); + let x91: u64 = u64::from(x90).wrapping_add(x70); let (x92, x93) = fiat_p384_addcarryx_u64(0x0, x57, x79); let (x94, x95) = fiat_p384_addcarryx_u64(x93, x59, x81); let (x96, x97) = fiat_p384_addcarryx_u64(x95, x61, x83); let (x98, x99) = fiat_p384_addcarryx_u64(x97, x63, x85); let (x100, x101) = fiat_p384_addcarryx_u64(x99, x65, x87); let (x102, x103) = fiat_p384_addcarryx_u64(x101, x67, x89); - let (x104, x105) = fiat_p384_addcarryx_u64(x103, (x68 as u64), x91); - let (x106, x107) = fiat_p384_mulx_u64(x92, 0x100000001); + let (x104, x105) = fiat_p384_addcarryx_u64(x103, u64::from(x68), x91); + let (x106, _x107) = fiat_p384_mulx_u64(x92, 0x100000001); let (x108, x109) = fiat_p384_mulx_u64(x106, 0xffffffffffffffff); let (x110, x111) = fiat_p384_mulx_u64(x106, 0xffffffffffffffff); let (x112, x113) = fiat_p384_mulx_u64(x106, 0xffffffffffffffff); @@ -397,27 +364,27 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x124, x125) = fiat_p384_addcarryx_u64(x123, x115, x112); let (x126, x127) = fiat_p384_addcarryx_u64(x125, x113, x110); let (x128, x129) = fiat_p384_addcarryx_u64(x127, x111, x108); - let x130: u64 = ((x129 as u64) + x109); - let (x131, x132) = fiat_p384_addcarryx_u64(0x0, x92, x118); + let x130: u64 = u64::from(x129).wrapping_add(x109); + let (_x131, x132) = fiat_p384_addcarryx_u64(0x0, x92, x118); let (x133, x134) = fiat_p384_addcarryx_u64(x132, x94, x120); let (x135, x136) = fiat_p384_addcarryx_u64(x134, x96, x122); let (x137, x138) = fiat_p384_addcarryx_u64(x136, x98, x124); let (x139, x140) = fiat_p384_addcarryx_u64(x138, x100, x126); let (x141, x142) = fiat_p384_addcarryx_u64(x140, x102, x128); let (x143, x144) = fiat_p384_addcarryx_u64(x142, x104, x130); - let x145: u64 = ((x144 as u64) + (x105 as u64)); - let (x146, x147) = fiat_p384_mulx_u64(x2, (arg1[5])); - let (x148, x149) = fiat_p384_mulx_u64(x2, (arg1[4])); - let (x150, x151) = fiat_p384_mulx_u64(x2, (arg1[3])); - let (x152, x153) = fiat_p384_mulx_u64(x2, (arg1[2])); - let (x154, x155) = fiat_p384_mulx_u64(x2, (arg1[1])); - let (x156, x157) = fiat_p384_mulx_u64(x2, (arg1[0])); + let x145: u64 = u64::from(x144).wrapping_add(u64::from(x105)); + let (x146, x147) = fiat_p384_mulx_u64(x2, arg1[5]); + let (x148, x149) = fiat_p384_mulx_u64(x2, arg1[4]); + let (x150, x151) = fiat_p384_mulx_u64(x2, arg1[3]); + let (x152, x153) = fiat_p384_mulx_u64(x2, arg1[2]); + let (x154, x155) = fiat_p384_mulx_u64(x2, arg1[1]); + let (x156, x157) = fiat_p384_mulx_u64(x2, arg1[0]); let (x158, x159) = fiat_p384_addcarryx_u64(0x0, x157, x154); let (x160, x161) = fiat_p384_addcarryx_u64(x159, x155, x152); let (x162, x163) = fiat_p384_addcarryx_u64(x161, x153, x150); let (x164, x165) = fiat_p384_addcarryx_u64(x163, x151, x148); let (x166, x167) = fiat_p384_addcarryx_u64(x165, x149, x146); - let x168: u64 = ((x167 as u64) + x147); + let x168: u64 = u64::from(x167).wrapping_add(x147); let (x169, x170) = fiat_p384_addcarryx_u64(0x0, x133, x156); let (x171, x172) = fiat_p384_addcarryx_u64(x170, x135, x158); let (x173, x174) = fiat_p384_addcarryx_u64(x172, x137, x160); @@ -425,7 +392,7 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x177, x178) = fiat_p384_addcarryx_u64(x176, x141, x164); let (x179, x180) = fiat_p384_addcarryx_u64(x178, x143, x166); let (x181, x182) = fiat_p384_addcarryx_u64(x180, x145, x168); - let (x183, x184) = fiat_p384_mulx_u64(x169, 0x100000001); + let (x183, _x184) = fiat_p384_mulx_u64(x169, 0x100000001); let (x185, x186) = fiat_p384_mulx_u64(x183, 0xffffffffffffffff); let (x187, x188) = fiat_p384_mulx_u64(x183, 0xffffffffffffffff); let (x189, x190) = fiat_p384_mulx_u64(x183, 0xffffffffffffffff); @@ -437,27 +404,27 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x201, x202) = fiat_p384_addcarryx_u64(x200, x192, x189); let (x203, x204) = fiat_p384_addcarryx_u64(x202, x190, x187); let (x205, x206) = fiat_p384_addcarryx_u64(x204, x188, x185); - let x207: u64 = ((x206 as u64) + x186); - let (x208, x209) = fiat_p384_addcarryx_u64(0x0, x169, x195); + let x207: u64 = u64::from(x206).wrapping_add(x186); + let (_x208, x209) = fiat_p384_addcarryx_u64(0x0, x169, x195); let (x210, x211) = fiat_p384_addcarryx_u64(x209, x171, x197); let (x212, x213) = fiat_p384_addcarryx_u64(x211, x173, x199); let (x214, x215) = fiat_p384_addcarryx_u64(x213, x175, x201); let (x216, x217) = fiat_p384_addcarryx_u64(x215, x177, x203); let (x218, x219) = fiat_p384_addcarryx_u64(x217, x179, x205); let (x220, x221) = fiat_p384_addcarryx_u64(x219, x181, x207); - let x222: u64 = ((x221 as u64) + (x182 as u64)); - let (x223, x224) = fiat_p384_mulx_u64(x3, (arg1[5])); - let (x225, x226) = fiat_p384_mulx_u64(x3, (arg1[4])); - let (x227, x228) = fiat_p384_mulx_u64(x3, (arg1[3])); - let (x229, x230) = fiat_p384_mulx_u64(x3, (arg1[2])); - let (x231, x232) = fiat_p384_mulx_u64(x3, (arg1[1])); - let (x233, x234) = fiat_p384_mulx_u64(x3, (arg1[0])); + let x222: u64 = u64::from(x221).wrapping_add(u64::from(x182)); + let (x223, x224) = fiat_p384_mulx_u64(x3, arg1[5]); + let (x225, x226) = fiat_p384_mulx_u64(x3, arg1[4]); + let (x227, x228) = fiat_p384_mulx_u64(x3, arg1[3]); + let (x229, x230) = fiat_p384_mulx_u64(x3, arg1[2]); + let (x231, x232) = fiat_p384_mulx_u64(x3, arg1[1]); + let (x233, x234) = fiat_p384_mulx_u64(x3, arg1[0]); let (x235, x236) = fiat_p384_addcarryx_u64(0x0, x234, x231); let (x237, x238) = fiat_p384_addcarryx_u64(x236, x232, x229); let (x239, x240) = fiat_p384_addcarryx_u64(x238, x230, x227); let (x241, x242) = fiat_p384_addcarryx_u64(x240, x228, x225); let (x243, x244) = fiat_p384_addcarryx_u64(x242, x226, x223); - let x245: u64 = ((x244 as u64) + x224); + let x245: u64 = u64::from(x244).wrapping_add(x224); let (x246, x247) = fiat_p384_addcarryx_u64(0x0, x210, x233); let (x248, x249) = fiat_p384_addcarryx_u64(x247, x212, x235); let (x250, x251) = fiat_p384_addcarryx_u64(x249, x214, x237); @@ -465,7 +432,7 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x254, x255) = fiat_p384_addcarryx_u64(x253, x218, x241); let (x256, x257) = fiat_p384_addcarryx_u64(x255, x220, x243); let (x258, x259) = fiat_p384_addcarryx_u64(x257, x222, x245); - let (x260, x261) = fiat_p384_mulx_u64(x246, 0x100000001); + let (x260, _x261) = fiat_p384_mulx_u64(x246, 0x100000001); let (x262, x263) = fiat_p384_mulx_u64(x260, 0xffffffffffffffff); let (x264, x265) = fiat_p384_mulx_u64(x260, 0xffffffffffffffff); let (x266, x267) = fiat_p384_mulx_u64(x260, 0xffffffffffffffff); @@ -477,27 +444,27 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x278, x279) = fiat_p384_addcarryx_u64(x277, x269, x266); let (x280, x281) = fiat_p384_addcarryx_u64(x279, x267, x264); let (x282, x283) = fiat_p384_addcarryx_u64(x281, x265, x262); - let x284: u64 = ((x283 as u64) + x263); - let (x285, x286) = fiat_p384_addcarryx_u64(0x0, x246, x272); + let x284: u64 = u64::from(x283).wrapping_add(x263); + let (_x285, x286) = fiat_p384_addcarryx_u64(0x0, x246, x272); let (x287, x288) = fiat_p384_addcarryx_u64(x286, x248, x274); let (x289, x290) = fiat_p384_addcarryx_u64(x288, x250, x276); let (x291, x292) = fiat_p384_addcarryx_u64(x290, x252, x278); let (x293, x294) = fiat_p384_addcarryx_u64(x292, x254, x280); let (x295, x296) = fiat_p384_addcarryx_u64(x294, x256, x282); let (x297, x298) = fiat_p384_addcarryx_u64(x296, x258, x284); - let x299: u64 = ((x298 as u64) + (x259 as u64)); - let (x300, x301) = fiat_p384_mulx_u64(x4, (arg1[5])); - let (x302, x303) = fiat_p384_mulx_u64(x4, (arg1[4])); - let (x304, x305) = fiat_p384_mulx_u64(x4, (arg1[3])); - let (x306, x307) = fiat_p384_mulx_u64(x4, (arg1[2])); - let (x308, x309) = fiat_p384_mulx_u64(x4, (arg1[1])); - let (x310, x311) = fiat_p384_mulx_u64(x4, (arg1[0])); + let x299: u64 = u64::from(x298).wrapping_add(u64::from(x259)); + let (x300, x301) = fiat_p384_mulx_u64(x4, arg1[5]); + let (x302, x303) = fiat_p384_mulx_u64(x4, arg1[4]); + let (x304, x305) = fiat_p384_mulx_u64(x4, arg1[3]); + let (x306, x307) = fiat_p384_mulx_u64(x4, arg1[2]); + let (x308, x309) = fiat_p384_mulx_u64(x4, arg1[1]); + let (x310, x311) = fiat_p384_mulx_u64(x4, arg1[0]); let (x312, x313) = fiat_p384_addcarryx_u64(0x0, x311, x308); let (x314, x315) = fiat_p384_addcarryx_u64(x313, x309, x306); let (x316, x317) = fiat_p384_addcarryx_u64(x315, x307, x304); let (x318, x319) = fiat_p384_addcarryx_u64(x317, x305, x302); let (x320, x321) = fiat_p384_addcarryx_u64(x319, x303, x300); - let x322: u64 = ((x321 as u64) + x301); + let x322: u64 = u64::from(x321).wrapping_add(x301); let (x323, x324) = fiat_p384_addcarryx_u64(0x0, x287, x310); let (x325, x326) = fiat_p384_addcarryx_u64(x324, x289, x312); let (x327, x328) = fiat_p384_addcarryx_u64(x326, x291, x314); @@ -505,7 +472,7 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x331, x332) = fiat_p384_addcarryx_u64(x330, x295, x318); let (x333, x334) = fiat_p384_addcarryx_u64(x332, x297, x320); let (x335, x336) = fiat_p384_addcarryx_u64(x334, x299, x322); - let (x337, x338) = fiat_p384_mulx_u64(x323, 0x100000001); + let (x337, _x338) = fiat_p384_mulx_u64(x323, 0x100000001); let (x339, x340) = fiat_p384_mulx_u64(x337, 0xffffffffffffffff); let (x341, x342) = fiat_p384_mulx_u64(x337, 0xffffffffffffffff); let (x343, x344) = fiat_p384_mulx_u64(x337, 0xffffffffffffffff); @@ -517,27 +484,27 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x355, x356) = fiat_p384_addcarryx_u64(x354, x346, x343); let (x357, x358) = fiat_p384_addcarryx_u64(x356, x344, x341); let (x359, x360) = fiat_p384_addcarryx_u64(x358, x342, x339); - let x361: u64 = ((x360 as u64) + x340); - let (x362, x363) = fiat_p384_addcarryx_u64(0x0, x323, x349); + let x361: u64 = u64::from(x360).wrapping_add(x340); + let (_x362, x363) = fiat_p384_addcarryx_u64(0x0, x323, x349); let (x364, x365) = fiat_p384_addcarryx_u64(x363, x325, x351); let (x366, x367) = fiat_p384_addcarryx_u64(x365, x327, x353); let (x368, x369) = fiat_p384_addcarryx_u64(x367, x329, x355); let (x370, x371) = fiat_p384_addcarryx_u64(x369, x331, x357); let (x372, x373) = fiat_p384_addcarryx_u64(x371, x333, x359); let (x374, x375) = fiat_p384_addcarryx_u64(x373, x335, x361); - let x376: u64 = ((x375 as u64) + (x336 as u64)); - let (x377, x378) = fiat_p384_mulx_u64(x5, (arg1[5])); - let (x379, x380) = fiat_p384_mulx_u64(x5, (arg1[4])); - let (x381, x382) = fiat_p384_mulx_u64(x5, (arg1[3])); - let (x383, x384) = fiat_p384_mulx_u64(x5, (arg1[2])); - let (x385, x386) = fiat_p384_mulx_u64(x5, (arg1[1])); - let (x387, x388) = fiat_p384_mulx_u64(x5, (arg1[0])); + let x376: u64 = u64::from(x375).wrapping_add(u64::from(x336)); + let (x377, x378) = fiat_p384_mulx_u64(x5, arg1[5]); + let (x379, x380) = fiat_p384_mulx_u64(x5, arg1[4]); + let (x381, x382) = fiat_p384_mulx_u64(x5, arg1[3]); + let (x383, x384) = fiat_p384_mulx_u64(x5, arg1[2]); + let (x385, x386) = fiat_p384_mulx_u64(x5, arg1[1]); + let (x387, x388) = fiat_p384_mulx_u64(x5, arg1[0]); let (x389, x390) = fiat_p384_addcarryx_u64(0x0, x388, x385); let (x391, x392) = fiat_p384_addcarryx_u64(x390, x386, x383); let (x393, x394) = fiat_p384_addcarryx_u64(x392, x384, x381); let (x395, x396) = fiat_p384_addcarryx_u64(x394, x382, x379); let (x397, x398) = fiat_p384_addcarryx_u64(x396, x380, x377); - let x399: u64 = ((x398 as u64) + x378); + let x399: u64 = u64::from(x398).wrapping_add(x378); let (x400, x401) = fiat_p384_addcarryx_u64(0x0, x364, x387); let (x402, x403) = fiat_p384_addcarryx_u64(x401, x366, x389); let (x404, x405) = fiat_p384_addcarryx_u64(x403, x368, x391); @@ -545,7 +512,7 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x408, x409) = fiat_p384_addcarryx_u64(x407, x372, x395); let (x410, x411) = fiat_p384_addcarryx_u64(x409, x374, x397); let (x412, x413) = fiat_p384_addcarryx_u64(x411, x376, x399); - let (x414, x415) = fiat_p384_mulx_u64(x400, 0x100000001); + let (x414, _x415) = fiat_p384_mulx_u64(x400, 0x100000001); let (x416, x417) = fiat_p384_mulx_u64(x414, 0xffffffffffffffff); let (x418, x419) = fiat_p384_mulx_u64(x414, 0xffffffffffffffff); let (x420, x421) = fiat_p384_mulx_u64(x414, 0xffffffffffffffff); @@ -557,28 +524,28 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x432, x433) = fiat_p384_addcarryx_u64(x431, x423, x420); let (x434, x435) = fiat_p384_addcarryx_u64(x433, x421, x418); let (x436, x437) = fiat_p384_addcarryx_u64(x435, x419, x416); - let x438: u64 = ((x437 as u64) + x417); - let (x439, x440) = fiat_p384_addcarryx_u64(0x0, x400, x426); + let x438: u64 = u64::from(x437).wrapping_add(x417); + let (_x439, x440) = fiat_p384_addcarryx_u64(0x0, x400, x426); let (x441, x442) = fiat_p384_addcarryx_u64(x440, x402, x428); let (x443, x444) = fiat_p384_addcarryx_u64(x442, x404, x430); let (x445, x446) = fiat_p384_addcarryx_u64(x444, x406, x432); let (x447, x448) = fiat_p384_addcarryx_u64(x446, x408, x434); let (x449, x450) = fiat_p384_addcarryx_u64(x448, x410, x436); let (x451, x452) = fiat_p384_addcarryx_u64(x450, x412, x438); - let x453: u64 = ((x452 as u64) + (x413 as u64)); + let x453: u64 = u64::from(x452).wrapping_add(u64::from(x413)); let (x454, x455) = fiat_p384_subborrowx_u64(0x0, x441, 0xffffffff); let (x456, x457) = fiat_p384_subborrowx_u64(x455, x443, 0xffffffff00000000); let (x458, x459) = fiat_p384_subborrowx_u64(x457, x445, 0xfffffffffffffffe); let (x460, x461) = fiat_p384_subborrowx_u64(x459, x447, 0xffffffffffffffff); let (x462, x463) = fiat_p384_subborrowx_u64(x461, x449, 0xffffffffffffffff); let (x464, x465) = fiat_p384_subborrowx_u64(x463, x451, 0xffffffffffffffff); - let (x466, x467) = fiat_p384_subborrowx_u64(x465, x453, (0x0 as u64)); - let (x468) = fiat_p384_cmovznz_u64(x467, x454, x441); - let (x469) = fiat_p384_cmovznz_u64(x467, x456, x443); - let (x470) = fiat_p384_cmovznz_u64(x467, x458, x445); - let (x471) = fiat_p384_cmovznz_u64(x467, x460, x447); - let (x472) = fiat_p384_cmovznz_u64(x467, x462, x449); - let (x473) = fiat_p384_cmovznz_u64(x467, x464, x451); + let (_x466, x467) = fiat_p384_subborrowx_u64(x465, x453, 0x0_u64); + let x468 = fiat_p384_cmovznz_u64(x467, x454, x441); + let x469 = fiat_p384_cmovznz_u64(x467, x456, x443); + let x470 = fiat_p384_cmovznz_u64(x467, x458, x445); + let x471 = fiat_p384_cmovznz_u64(x467, x460, x447); + let x472 = fiat_p384_cmovznz_u64(x467, x462, x449); + let x473 = fiat_p384_cmovznz_u64(x467, x464, x451); out1[0] = x468; out1[1] = x469; out1[2] = x470; diff --git a/src/auth/ecdsa_x86_64_asm.rs b/src/auth/ecdsa_x86_64_asm.rs index 79e0dcbe..8349f44b 100644 --- a/src/auth/ecdsa_x86_64_asm.rs +++ b/src/auth/ecdsa_x86_64_asm.rs @@ -5,8 +5,6 @@ //! module owns the ABI boundary; `ecdsa.rs` owns scalar validation, blinding, //! affine conversion, and public API semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; use super::ZeroizingWords; diff --git a/src/auth/ed25519.rs b/src/auth/ed25519.rs index 2733617c..980cc4a7 100644 --- a/src/auth/ed25519.rs +++ b/src/auth/ed25519.rs @@ -429,7 +429,7 @@ pub struct Ed25519Keypair { #[cfg(feature = "diag")] #[doc(hidden)] -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub struct DiagEd25519VerifyScalars { pub r_bytes: [u8; PUBLIC_KEY_LENGTH], pub s_canonical: [u8; SECRET_KEY_LENGTH], @@ -992,24 +992,8 @@ mod tests { } fn decode_hex(hex: &str) -> [u8; N] { - let bytes = hex.as_bytes(); let mut out = [0u8; N]; - - for (dst, chunk) in out.iter_mut().zip(bytes.chunks_exact(2)) { - *dst = match chunk[0] { - b'0'..=b'9' => chunk[0] - b'0', - b'a'..=b'f' => chunk[0] - b'a' + 10, - b'A'..=b'F' => chunk[0] - b'A' + 10, - _ => panic!("invalid hex"), - } << 4 - | match chunk[1] { - b'0'..=b'9' => chunk[1] - b'0', - b'a'..=b'f' => chunk[1] - b'a' + 10, - b'A'..=b'F' => chunk[1] - b'A' + 10, - _ => panic!("invalid hex"), - }; - } - + crate::hex::from_hex(hex, &mut out).expect("RFC 8032 vectors must contain valid fixed-width hexadecimal"); out } @@ -1033,8 +1017,12 @@ mod tests { let signature = keypair.sign(message); assert_eq!(secret.sign(message), signature); - assert!(keypair.public_key().verify(message, &signature).is_ok()); - assert!(verify(message, &keypair.public_key(), &signature).is_ok()); + keypair + .public_key() + .verify(message, &signature) + .expect("the keypair signature must verify with its public key"); + verify(message, &keypair.public_key(), &signature) + .expect("the keypair signature must verify through the free function"); } #[test] @@ -1189,7 +1177,7 @@ mod tests { for scalar_bytes in [[0u8; Ed25519SecretKey::LENGTH], one, reduced] { let portable = point::ExtendedPoint::scalar_mul_basepoint(&scalar_bytes) .to_bytes() - .unwrap(); + .expect("basepoint multiplication produces an encodable Edwards point"); let asm = super::basepoint_mul_encoded_dispatch(&scalar_bytes); assert_eq!(asm, portable); @@ -1203,7 +1191,12 @@ mod tests { { let (asm_public, asm_point) = super::x86_64_asm::basepoint_mul_public(&scalar_bytes); assert_eq!(asm_public, portable); - assert_eq!(asm_point.to_bytes().unwrap(), portable); + assert_eq!( + asm_point + .to_bytes() + .expect("assembly basepoint multiplication produces an encodable Edwards point"), + portable + ); } } } @@ -1225,8 +1218,11 @@ mod tests { let public = keypair.public_key(); let signature = keypair.sign(message); let (r_bytes, s_bytes) = super::split_signature(&signature); - let a_point = public.point().unwrap(); - let s = scalar::from_canonical_bytes(&s_bytes).unwrap(); + let a_point = public + .point() + .expect("a locally generated Ed25519 public key decodes to an Edwards point"); + let s = + scalar::from_canonical_bytes(&s_bytes).expect("a locally generated Ed25519 signature has a canonical scalar"); let challenge_digest = super::hash_challenge(&r_bytes, public.as_bytes(), message); let challenge = scalar::reduce_bytes_mod_order(&challenge_digest); @@ -1236,10 +1232,10 @@ mod tests { let portable = point::straus_wnaf_basepoint_vartime(&s_canonical, &neg_challenge_bytes, &a_point) .to_bytes() - .unwrap(); + .expect("double-scalar multiplication produces an encodable Edwards point"); let asm = super::aarch64_asm::double_scalar_basepoint_encoded(&s_canonical, &neg_challenge_bytes, public.as_bytes()) - .unwrap(); + .expect("the assembly double-scalar path accepts a locally generated public key"); assert_eq!(asm, portable); assert_eq!(asm, r_bytes); diff --git a/src/auth/ed25519/aarch64_asm.rs b/src/auth/ed25519/aarch64_asm.rs index ef8d5c08..e35a2378 100644 --- a/src/auth/ed25519/aarch64_asm.rs +++ b/src/auth/ed25519/aarch64_asm.rs @@ -5,8 +5,6 @@ //! backends. This module owns the ABI boundary; `ed25519.rs` owns //! public-key/signature validation semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; use super::constants::{PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; diff --git a/src/auth/ed25519/basepoint_table_ifma.rs b/src/auth/ed25519/basepoint_table_ifma.rs index 316b1e8b..f92d98e0 100644 --- a/src/auth/ed25519/basepoint_table_ifma.rs +++ b/src/auth/ed25519/basepoint_table_ifma.rs @@ -6,8 +6,8 @@ //! Lanes: `(d2·(Y−X), d2·(Y+X), 2·d2·Z, −2·d1·T)`. //! //! Generated from the canonical Ed25519 basepoint using portable scalar math. -//! Validated by `gen_basepoint_wnaf8_ifma_table` test in `point.rs`. -#![allow(clippy::unreadable_literal)] +//! Validated by `ifma_wnaf8_table_matches_portable_odd_multiples` in +//! `point_avx2.rs`. pub(crate) static BASEPOINT_WNAF8_IFMA_RAW: [[[i64; 4]; 5]; 64] = [ // 1B diff --git a/src/auth/ed25519/field_avx2.rs b/src/auth/ed25519/field_avx2.rs index 5384e270..69a8ddad 100644 --- a/src/auth/ed25519/field_avx2.rs +++ b/src/auth/ed25519/field_avx2.rs @@ -31,9 +31,7 @@ use super::field::FieldElement; const LOW_25_BITS: i64 = (1 << 25) - 1; const LOW_26_BITS: i64 = (1 << 26) - 1; - -/// D-lane blend mask: positions 5 and 7 in each `__m256i` (u32 view). -const D_BLEND: i32 = 0b1010_0000u8 as i32; +const LOW_26_BITS_U64: u64 = (1 << 26) - 1; /// Four field elements packed for AVX2 parallel processing. /// @@ -49,30 +47,32 @@ pub(crate) struct FieldElement2625x4(pub(crate) [__m256i; 5]); /// Lane rearrangement patterns for `shuffle`. #[derive(Clone, Copy)] #[repr(u8)] -#[allow(clippy::upper_case_acronyms, dead_code)] // Lane labels, complete API. pub(crate) enum Shuffle { - /// Identity: (A, B, C, D) → (A, B, C, D) - ABCD, /// Swap pairs: (A, B, C, D) → (B, A, D, C) - BADC, + SwapPairs, /// Swap left pair only: (A, B, C, D) → (B, A, C, D) - BACD, + SwapAB, /// Swap right pair only: (A, B, C, D) → (A, B, D, C) - ABDC, + SwapCD, /// Broadcast A: (A, B, C, D) → (A, A, A, A) - AAAA, + #[cfg(any(test, feature = "ed25519"))] + BroadcastA, /// Broadcast B: (A, B, C, D) → (B, B, B, B) - BBBB, + #[cfg(any(test, feature = "ed25519"))] + BroadcastB, /// (A, B, C, D) → (C, A, C, A) - CACA, + #[cfg(any(test, feature = "ed25519"))] + AlternateCA, /// (A, B, C, D) → (D, B, B, D) - DBBD, + #[cfg(any(test, feature = "ed25519"))] + OuterDInnerB, /// (A, B, C, D) → (A, D, D, A) - ADDA, + OuterAInnerD, /// (A, B, C, D) → (C, B, C, B) - CBCB, + AlternateCB, /// (A, B, C, D) → (A, B, A, B) - ABAB, + #[cfg(any(test, feature = "ed25519"))] + RepeatAB, } impl Shuffle { @@ -80,17 +80,21 @@ impl Shuffle { #[inline(always)] fn control(self) -> [i32; 8] { match self { - Self::ABCD => [0, 1, 2, 3, 4, 5, 6, 7], - Self::BADC => [1, 0, 3, 2, 5, 4, 7, 6], - Self::BACD => [1, 0, 3, 2, 4, 5, 6, 7], - Self::ABDC => [0, 1, 2, 3, 5, 4, 7, 6], - Self::AAAA => [0, 0, 2, 2, 0, 0, 2, 2], - Self::BBBB => [1, 1, 3, 3, 1, 1, 3, 3], - Self::CACA => [4, 0, 6, 2, 4, 0, 6, 2], - Self::DBBD => [5, 1, 7, 3, 1, 5, 3, 7], - Self::ADDA => [0, 5, 2, 7, 5, 0, 7, 2], - Self::CBCB => [4, 1, 6, 3, 4, 1, 6, 3], - Self::ABAB => [0, 1, 2, 3, 0, 1, 2, 3], + Self::SwapPairs => [1, 0, 3, 2, 5, 4, 7, 6], + Self::SwapAB => [1, 0, 3, 2, 4, 5, 6, 7], + Self::SwapCD => [0, 1, 2, 3, 5, 4, 7, 6], + #[cfg(any(test, feature = "ed25519"))] + Self::BroadcastA => [0, 0, 2, 2, 0, 0, 2, 2], + #[cfg(any(test, feature = "ed25519"))] + Self::BroadcastB => [1, 1, 3, 3, 1, 1, 3, 3], + #[cfg(any(test, feature = "ed25519"))] + Self::AlternateCA => [4, 0, 6, 2, 4, 0, 6, 2], + #[cfg(any(test, feature = "ed25519"))] + Self::OuterDInnerB => [5, 1, 7, 3, 1, 5, 3, 7], + Self::OuterAInnerD => [0, 5, 2, 7, 5, 0, 7, 2], + Self::AlternateCB => [4, 1, 6, 3, 4, 1, 6, 3], + #[cfg(any(test, feature = "ed25519"))] + Self::RepeatAB => [0, 1, 2, 3, 0, 1, 2, 3], } } } @@ -103,13 +107,9 @@ impl Shuffle { /// Lane positions: `[a_even, b_even, a_odd, b_odd, c_even, d_even, c_odd, d_odd]`. #[derive(Clone, Copy)] #[repr(u8)] -#[allow(clippy::upper_case_acronyms, dead_code)] // Lane labels, complete API. pub(crate) enum Lanes { - /// Select A lanes: positions 0, 2 - A = 0b0000_0101, - /// Select B lanes: positions 1, 3 - B = 0b0000_1010, /// Select C lanes: positions 4, 6 + #[cfg(any(test, feature = "ed25519"))] C = 0b0101_0000, /// Select D lanes: positions 5, 7 D = 0b1010_0000, @@ -118,15 +118,14 @@ pub(crate) enum Lanes { /// Select A and C lanes: positions 0, 2, 4, 6 AC = 0b0101_0101, /// Select A and D lanes: positions 0, 2, 5, 7 + #[cfg(any(test, feature = "ed25519"))] AD = 0b1010_0101, /// Select B and C lanes: positions 1, 3, 4, 6 + #[cfg(any(test, feature = "ed25519"))] BC = 0b0101_1010, - /// Select B, C, and D lanes: positions 1, 3, 4, 5, 6, 7 - BCD = 0b1111_1010, - /// Select C and D lanes: positions 4-7 - CD = 0b1111_0000, - /// Select all lanes - ABCD = 0b1111_1111, + /// Select every lane except A: positions 1, 3, 4, 5, 6, 7 + #[cfg(any(test, feature = "ed25519"))] + ExceptA = 0b1111_1010, } /// Zero-extend packed u32 pairs into two u64x4 vectors suitable for `vpmuludq`. @@ -138,11 +137,10 @@ pub(crate) enum Lanes { /// /// # Safety /// -/// Caller must ensure AVX2 is available. +/// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn unpack_pair(v: __m256i) -> (__m256i, __m256i) { +fn unpack_pair(v: __m256i) -> (__m256i, __m256i) { let zero = _mm256_setzero_si256(); let lo = _mm256_unpacklo_epi32(v, zero); let hi = _mm256_unpackhi_epi32(v, zero); @@ -155,11 +153,10 @@ unsafe fn unpack_pair(v: __m256i) -> (__m256i, __m256i) { /// /// # Safety /// -/// Caller must ensure AVX2 is available. +/// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn repack_pair(lo: __m256i, hi: __m256i) -> __m256i { +fn repack_pair(lo: __m256i, hi: __m256i) -> __m256i { // Shuffle to pack the low u32 of each 64-bit lane into consecutive positions. // _mm256_shuffle_epi32 with imm [0, 2, 0, 2] = 0b10_00_10_00 packs positions // 0 and 2 within each 128-bit lane. @@ -173,11 +170,10 @@ unsafe fn repack_pair(lo: __m256i, hi: __m256i) -> __m256i { /// /// # Safety /// -/// Caller must ensure AVX2 is available. +/// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn mul32(a: __m256i, b: __m256i) -> __m256i { +fn mul32(a: __m256i, b: __m256i) -> __m256i { _mm256_mul_epu32(a, b) } @@ -185,14 +181,19 @@ unsafe fn mul32(a: __m256i, b: __m256i) -> __m256i { /// /// # Safety /// -/// Caller must ensure AVX2 is available. +/// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add64(a: __m256i, b: __m256i) -> __m256i { +fn add64(a: __m256i, b: __m256i) -> __m256i { _mm256_add_epi64(a, b) } +#[inline(always)] +fn low_word_as_i32(value: u64) -> i32 { + let bytes = value.to_le_bytes(); + i32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) +} + // FieldElement2625x4 implementation #[cfg(target_arch = "x86_64")] @@ -201,11 +202,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn zero() -> Self { + pub(crate) fn zero() -> Self { Self([_mm256_setzero_si256(); 5]) } @@ -216,42 +216,43 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn new(a: &FieldElement, b: &FieldElement, c: &FieldElement, d: &FieldElement) -> Self { + pub(crate) fn new(a: &FieldElement, b: &FieldElement, c: &FieldElement, d: &FieldElement) -> Self { let al = a.limbs(); let bl = b.limbs(); let cl = c.limbs(); let dl = d.limbs(); - let mask = LOW_26_BITS as u64; let out = [ - Self::pack_limb_pair(al[0], bl[0], cl[0], dl[0], mask), - Self::pack_limb_pair(al[1], bl[1], cl[1], dl[1], mask), - Self::pack_limb_pair(al[2], bl[2], cl[2], dl[2], mask), - Self::pack_limb_pair(al[3], bl[3], cl[3], dl[3], mask), - Self::pack_limb_pair(al[4], bl[4], cl[4], dl[4], mask), + Self::pack_limb_pair(al[0], bl[0], cl[0], dl[0]), + Self::pack_limb_pair(al[1], bl[1], cl[1], dl[1]), + Self::pack_limb_pair(al[2], bl[2], cl[2], dl[2]), + Self::pack_limb_pair(al[3], bl[3], cl[3], dl[3]), + Self::pack_limb_pair(al[4], bl[4], cl[4], dl[4]), ]; // Odd limbs from a non-reduced FieldElement may exceed 25 bits. Self(out).reduce() } + /// # Safety + /// + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - unsafe fn pack_limb_pair(al: u64, bl: u64, cl: u64, dl: u64, mask: u64) -> __m256i { + fn pack_limb_pair(al: u64, bl: u64, cl: u64, dl: u64) -> __m256i { + // Valid radix-51 limbs stay below 2^55, so each shifted half fits in an unsigned 32-bit lane. _mm256_setr_epi32( - (al & mask) as i32, - (bl & mask) as i32, - (al >> 26) as i32, - (bl >> 26) as i32, - (cl & mask) as i32, - (dl & mask) as i32, - (cl >> 26) as i32, - (dl >> 26) as i32, + low_word_as_i32(al & LOW_26_BITS_U64), + low_word_as_i32(bl & LOW_26_BITS_U64), + low_word_as_i32(al >> 26), + low_word_as_i32(bl >> 26), + low_word_as_i32(cl & LOW_26_BITS_U64), + low_word_as_i32(dl & LOW_26_BITS_U64), + low_word_as_i32(cl >> 26), + low_word_as_i32(dl >> 26), ) } @@ -259,11 +260,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn split(&self) -> [FieldElement; 4] { + pub(crate) fn split(&self) -> [FieldElement; 4] { let mut al = [0u64; 5]; let mut bl = [0u64; 5]; let mut cl = [0u64; 5]; @@ -277,7 +277,8 @@ impl FieldElement2625x4 { .zip(self.0.iter()) { let mut tmp = [0u32; 8]; - _mm256_storeu_si256(tmp.as_mut_ptr().cast(), *vec); + // SAFETY: AVX2 is active in this function, and `tmp` provides 32 writable bytes for the unaligned store. + unsafe { _mm256_storeu_si256(tmp.as_mut_ptr().cast(), *vec) }; *a_out = u64::from(tmp[0]) | (u64::from(tmp[2]) << 26); *b_out = u64::from(tmp[1]) | (u64::from(tmp[3]) << 26); @@ -302,11 +303,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn add(&self, rhs: &Self) -> Self { + pub(crate) fn add(&self, rhs: &Self) -> Self { Self([ _mm256_add_epi32(self.0[0], rhs.0[0]), _mm256_add_epi32(self.0[1], rhs.0[1]), @@ -323,35 +323,34 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn sub(&self, rhs: &Self) -> Self { + pub(crate) fn sub(&self, rhs: &Self) -> Self { // 2p in radix 10×(26/25): // p = 2^255 - 19 // Limb 0: 2 * ((1 << 26) - 19) = 2^27 - 38 // Even limbs 2,4,6,8: 2 * ((1 << 26) - 1) = 2^27 - 2 // Odd limbs 1,3,5,7,9: 2 * ((1 << 25) - 1) = 2^26 - 2 let bias_0 = _mm256_setr_epi32( - (2 * ((1i64 << 26) - 19)) as i32, - (2 * ((1i64 << 26) - 19)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, - (2 * ((1i64 << 26) - 19)) as i32, - (2 * ((1i64 << 26) - 19)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, + (1 << 27) - 38, + (1 << 27) - 38, + (1 << 26) - 2, + (1 << 26) - 2, + (1 << 27) - 38, + (1 << 27) - 38, + (1 << 26) - 2, + (1 << 26) - 2, ); let bias_n = _mm256_setr_epi32( - (2 * ((1i64 << 26) - 1)) as i32, - (2 * ((1i64 << 26) - 1)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, - (2 * ((1i64 << 26) - 1)) as i32, - (2 * ((1i64 << 26) - 1)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, + (1 << 27) - 2, + (1 << 27) - 2, + (1 << 26) - 2, + (1 << 26) - 2, + (1 << 27) - 2, + (1 << 27) - 2, + (1 << 26) - 2, + (1 << 26) - 2, ); Self([ @@ -370,11 +369,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn negate_lazy(&self) -> Self { + pub(crate) fn negate_lazy(&self) -> Self { Self::zero().sub(self) } @@ -387,11 +385,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn shuffle(&self, pattern: Shuffle) -> Self { + pub(crate) fn shuffle(&self, pattern: Shuffle) -> Self { let ctrl = pattern.control(); let c = _mm256_setr_epi32(ctrl[0], ctrl[1], ctrl[2], ctrl[3], ctrl[4], ctrl[5], ctrl[6], ctrl[7]); Self([ @@ -410,11 +407,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn blend(&self, other: &Self, lanes: Lanes) -> Self { + pub(crate) fn blend(&self, other: &Self, lanes: Lanes) -> Self { // _mm256_blend_epi32 requires a compile-time immediate, so dispatch // on the enum variant. macro_rules! do_blend { @@ -430,17 +426,17 @@ impl FieldElement2625x4 { } match lanes { - Lanes::A => do_blend!(0b0000_0101), - Lanes::B => do_blend!(0b0000_1010), + #[cfg(any(test, feature = "ed25519"))] Lanes::C => do_blend!(0b0101_0000), Lanes::D => do_blend!(0b1010_0000), Lanes::AB => do_blend!(0b0000_1111), Lanes::AC => do_blend!(0b0101_0101), + #[cfg(any(test, feature = "ed25519"))] Lanes::AD => do_blend!(0b1010_0101), + #[cfg(any(test, feature = "ed25519"))] Lanes::BC => do_blend!(0b0101_1010), - Lanes::BCD => do_blend!(0b1111_1010), - Lanes::CD => do_blend!(0b1111_0000), - Lanes::ABCD => do_blend!(0b1111_1111), + #[cfg(any(test, feature = "ed25519"))] + Lanes::ExceptA => do_blend!(0b1111_1010), } } @@ -448,12 +444,11 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn select_mask(&self, other: &Self, mask: u64) -> Self { - let mask = _mm256_set1_epi64x(mask as i64); + pub(crate) fn select_mask(&self, other: &Self, mask: u64) -> Self { + let mask = _mm256_set1_epi64x(i64::from_ne_bytes(mask.to_ne_bytes())); Self([ _mm256_xor_si256( self.0[0], @@ -485,12 +480,11 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn diff_sum(&self) -> Self { - let swapped = self.shuffle(Shuffle::BADC); // (B, A, D, C) + pub(crate) fn diff_sum(&self) -> Self { + let swapped = self.shuffle(Shuffle::SwapPairs); // (B, A, D, C) let negated = self.negate_lazy(); // (-A, -B, -C, -D) let neg_ac = self.blend(&negated, Lanes::AC); // (-A, B, -C, D) swapped.add(&neg_ac) // (B-A, A+B, D-C, C+D) @@ -506,11 +500,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn reduce(&self) -> Self { + pub(crate) fn reduce(&self) -> Self { // Unpack to 10 × u64x4 for carry propagation. let (z0, z1) = unpack_pair(self.0[0]); let (z2, z3) = unpack_pair(self.0[1]); @@ -528,11 +521,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - unsafe fn reduce64(z: &mut [__m256i; 10]) -> Self { + fn reduce64(z: &mut [__m256i; 10]) -> Self { let mask_26 = _mm256_set1_epi64x(LOW_26_BITS); let mask_25 = _mm256_set1_epi64x(LOW_25_BITS); let v19 = _mm256_set1_epi64x(19); @@ -608,11 +600,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn mul(&self, rhs: &Self) -> Self { + pub(crate) fn mul(&self, rhs: &Self) -> Self { let v19 = _mm256_set1_epi64x(19); // Unpack self into 10 �� u64x4 (zero-extended for vpmuludq) @@ -790,11 +781,11 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - unsafe fn square_accum(&self) -> [__m256i; 10] { + #[cfg(any(test, feature = "ed25519"))] + fn square_accum(&self) -> [__m256i; 10] { let v19 = _mm256_set1_epi64x(19); let (x0, x1) = unpack_pair(self.0[0]); @@ -916,12 +907,11 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[cfg(test)] #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn square(&self) -> Self { + pub(crate) fn square(&self) -> Self { let mut z = self.square_accum(); Self::reduce64(&mut z) } @@ -940,11 +930,11 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn square_and_negate_d(&self) -> Self { + #[cfg(any(test, feature = "ed25519"))] + pub(crate) fn square_and_negate_d(&self) -> Self { let mut z = self.square_accum(); Self::negate_d_accum(&mut z); Self::reduce64(&mut z) @@ -957,10 +947,14 @@ impl FieldElement2625x4 { /// min bias ≈ 2^62) and small enough to fit in u64. The subsequent /// `reduce64` carry chain processes the biased values normally, producing /// a fully reduced negation with b < 0.007. + /// + /// # Safety + /// + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - unsafe fn negate_d_accum(z: &mut [__m256i; 10]) { + #[cfg(any(test, feature = "ed25519"))] + fn negate_d_accum(z: &mut [__m256i; 10]) { // p × 2^37 per limb (radix-26/25): let bias_even_0 = _mm256_set1_epi64x(((1i64 << 26) - 19) << 37); let bias_even = _mm256_set1_epi64x(((1i64 << 26) - 1) << 37); @@ -987,37 +981,6 @@ impl FieldElement2625x4 { neg_d!(8, bias_even); neg_d!(9, bias_odd); } - - /// Negate the D lane of each limb via `2p − D` in the packed u32 domain. - /// - /// **Not used by AVX2 `square_and_negate_d`** (which negates in u64 domain - /// for tighter bounds). Retained for potential external callers. - #[inline] - #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn, dead_code)] - unsafe fn negate_d_lane(fe: &mut Self) { - let p2_limb0_even = (2i64.wrapping_mul((1i64 << 26) - 19)) as i32; - let p2_limb_even = (2i64.wrapping_mul((1i64 << 26) - 1)) as i32; - let p2_limb_odd = (2i64.wrapping_mul((1i64 << 25) - 1)) as i32; - - let bias_0 = _mm256_setr_epi32(0, 0, 0, 0, 0, p2_limb0_even, 0, p2_limb_odd); - let bias_n = _mm256_setr_epi32(0, 0, 0, 0, 0, p2_limb_even, 0, p2_limb_odd); - - let neg0 = _mm256_sub_epi32(bias_0, fe.0[0]); - fe.0[0] = _mm256_blend_epi32::(fe.0[0], neg0); - - let neg1 = _mm256_sub_epi32(bias_n, fe.0[1]); - fe.0[1] = _mm256_blend_epi32::(fe.0[1], neg1); - - let neg2 = _mm256_sub_epi32(bias_n, fe.0[2]); - fe.0[2] = _mm256_blend_epi32::(fe.0[2], neg2); - - let neg3 = _mm256_sub_epi32(bias_n, fe.0[3]); - fe.0[3] = _mm256_blend_epi32::(fe.0[3], neg3); - - let neg4 = _mm256_sub_epi32(bias_n, fe.0[4]); - fe.0[4] = _mm256_blend_epi32::(fe.0[4], neg4); - } } // Tests @@ -1229,7 +1192,7 @@ mod tests { // SAFETY: AVX2 availability checked by the runtime guard above. unsafe { let packed = FieldElement2625x4::new(&a, &b, &c, &d); - let shuffled = packed.shuffle(Shuffle::BADC); + let shuffled = packed.shuffle(Shuffle::SwapPairs); let [ra, rb, rc, rd] = shuffled.split(); assert_eq!(ra.normalize(), b.normalize(), "BADC: A should be B"); diff --git a/src/auth/ed25519/field_ifma.rs b/src/auth/ed25519/field_ifma.rs index d27c7730..ab47b915 100644 --- a/src/auth/ed25519/field_ifma.rs +++ b/src/auth/ed25519/field_ifma.rs @@ -16,8 +16,8 @@ //! //! # Arithmetic convention //! -//! Field arithmetic is modular math (mod 2^255 - 19). Per CLAUDE.md rules, -//! `wrapping_*` is the correct choice for intentional modular arithmetic. +//! Field arithmetic is modular math (mod 2^255 - 19). Wrapping arithmetic is +//! reserved for operations whose field representation is intentionally modular. #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; @@ -60,8 +60,7 @@ pub(crate) struct FieldElement51x4(pub(crate) [__m256i; 5]); /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn mul19(x: __m256i) -> __m256i { +fn mul19(x: __m256i) -> __m256i { let x16 = _mm256_slli_epi64::<4>(x); let x2 = _mm256_slli_epi64::<1>(x); _mm256_add_epi64(_mm256_add_epi64(x16, x2), x) @@ -74,8 +73,7 @@ unsafe fn mul19(x: __m256i) -> __m256i { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn madd52lo(acc: __m256i, a: __m256i, b: __m256i) -> __m256i { +fn madd52lo(acc: __m256i, a: __m256i, b: __m256i) -> __m256i { _mm256_madd52lo_epu64(acc, a, b) } @@ -86,8 +84,7 @@ unsafe fn madd52lo(acc: __m256i, a: __m256i, b: __m256i) -> __m256i { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn madd52hi(acc: __m256i, a: __m256i, b: __m256i) -> __m256i { +fn madd52hi(acc: __m256i, a: __m256i, b: __m256i) -> __m256i { _mm256_madd52hi_epu64(acc, a, b) } @@ -102,13 +99,17 @@ unsafe fn madd52hi(acc: __m256i, a: __m256i, b: __m256i) -> __m256i { #[cfg(test)] #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn select_by_bit(bit: __m256i, val: __m256i) -> __m256i { +fn select_by_bit(bit: __m256i, val: __m256i) -> __m256i { // 0 → 0, 1 → 0xFFFF_FFFF_FFFF_FFFF let mask = _mm256_sub_epi64(_mm256_setzero_si256(), bit); _mm256_and_si256(mask, val) } +#[inline(always)] +fn u64_as_i64_bits(value: u64) -> i64 { + i64::from_ne_bytes(value.to_ne_bytes()) +} + // FieldElement51x4 implementation #[cfg(target_arch = "x86_64")] @@ -120,8 +121,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn zero() -> Self { + pub(crate) fn zero() -> Self { Self([_mm256_setzero_si256(); 5]) } @@ -132,19 +132,43 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn new(a: &FieldElement, b: &FieldElement, c: &FieldElement, d: &FieldElement) -> Self { + pub(crate) fn new(a: &FieldElement, b: &FieldElement, c: &FieldElement, d: &FieldElement) -> Self { let al = a.limbs(); let bl = b.limbs(); let cl = c.limbs(); let dl = d.limbs(); Self([ - _mm256_set_epi64x(dl[0] as i64, cl[0] as i64, bl[0] as i64, al[0] as i64), - _mm256_set_epi64x(dl[1] as i64, cl[1] as i64, bl[1] as i64, al[1] as i64), - _mm256_set_epi64x(dl[2] as i64, cl[2] as i64, bl[2] as i64, al[2] as i64), - _mm256_set_epi64x(dl[3] as i64, cl[3] as i64, bl[3] as i64, al[3] as i64), - _mm256_set_epi64x(dl[4] as i64, cl[4] as i64, bl[4] as i64, al[4] as i64), + _mm256_set_epi64x( + u64_as_i64_bits(dl[0]), + u64_as_i64_bits(cl[0]), + u64_as_i64_bits(bl[0]), + u64_as_i64_bits(al[0]), + ), + _mm256_set_epi64x( + u64_as_i64_bits(dl[1]), + u64_as_i64_bits(cl[1]), + u64_as_i64_bits(bl[1]), + u64_as_i64_bits(al[1]), + ), + _mm256_set_epi64x( + u64_as_i64_bits(dl[2]), + u64_as_i64_bits(cl[2]), + u64_as_i64_bits(bl[2]), + u64_as_i64_bits(al[2]), + ), + _mm256_set_epi64x( + u64_as_i64_bits(dl[3]), + u64_as_i64_bits(cl[3]), + u64_as_i64_bits(bl[3]), + u64_as_i64_bits(al[3]), + ), + _mm256_set_epi64x( + u64_as_i64_bits(dl[4]), + u64_as_i64_bits(cl[4]), + u64_as_i64_bits(bl[4]), + u64_as_i64_bits(al[4]), + ), ]) } @@ -155,8 +179,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn split(&self) -> [FieldElement; 4] { + pub(crate) fn split(&self) -> [FieldElement; 4] { let mut al = [0u64; 5]; let mut bl = [0u64; 5]; let mut cl = [0u64; 5]; @@ -169,7 +192,8 @@ impl FieldElement51x4 { .zip(self.0.iter()) { let mut tmp = [0u64; 4]; - _mm256_storeu_si256(tmp.as_mut_ptr().cast(), *vec); + // SAFETY: AVX2 is active in this function, and `tmp` provides 32 writable bytes for the unaligned store. + unsafe { _mm256_storeu_si256(tmp.as_mut_ptr().cast(), *vec) }; *a_out = tmp[0]; *b_out = tmp[1]; *c_out = tmp[2]; @@ -191,8 +215,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn add(&self, rhs: &Self) -> Self { + pub(crate) fn add(&self, rhs: &Self) -> Self { Self([ _mm256_add_epi64(self.0[0], rhs.0[0]), _mm256_add_epi64(self.0[1], rhs.0[1]), @@ -209,8 +232,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn sub(&self, rhs: &Self) -> Self { + pub(crate) fn sub(&self, rhs: &Self) -> Self { let bias_0 = _mm256_set1_epi64x(BIAS_0); let bias_n = _mm256_set1_epi64x(BIAS_N); @@ -230,8 +252,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn negate_lazy(&self) -> Self { + pub(crate) fn negate_lazy(&self) -> Self { Self::zero().sub(self) } @@ -244,8 +265,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn shuffle(&self, pattern: Shuffle) -> Self { + pub(crate) fn shuffle(&self, pattern: Shuffle) -> Self { // _mm256_permute4x64_epi64 requires a compile-time immediate. // IMM8 = (d_src << 6) | (c_src << 4) | (b_src << 2) | a_src macro_rules! do_shuffle { @@ -261,17 +281,21 @@ impl FieldElement51x4 { } match pattern { - Shuffle::ABCD => do_shuffle!(0b11_10_01_00), - Shuffle::BADC => do_shuffle!(0b10_11_00_01), - Shuffle::BACD => do_shuffle!(0b11_10_00_01), - Shuffle::ABDC => do_shuffle!(0b10_11_01_00), - Shuffle::AAAA => do_shuffle!(0b00_00_00_00), - Shuffle::BBBB => do_shuffle!(0b01_01_01_01), - Shuffle::CACA => do_shuffle!(0b00_10_00_10), - Shuffle::DBBD => do_shuffle!(0b11_01_01_11), - Shuffle::ADDA => do_shuffle!(0b00_11_11_00), - Shuffle::CBCB => do_shuffle!(0b01_10_01_10), - Shuffle::ABAB => do_shuffle!(0b01_00_01_00), + Shuffle::SwapPairs => do_shuffle!(0b10_11_00_01), + Shuffle::SwapAB => do_shuffle!(0b11_10_00_01), + Shuffle::SwapCD => do_shuffle!(0b10_11_01_00), + #[cfg(any(test, feature = "ed25519"))] + Shuffle::BroadcastA => do_shuffle!(0b00_00_00_00), + #[cfg(any(test, feature = "ed25519"))] + Shuffle::BroadcastB => do_shuffle!(0b01_01_01_01), + #[cfg(any(test, feature = "ed25519"))] + Shuffle::AlternateCA => do_shuffle!(0b00_10_00_10), + #[cfg(any(test, feature = "ed25519"))] + Shuffle::OuterDInnerB => do_shuffle!(0b11_01_01_11), + Shuffle::OuterAInnerD => do_shuffle!(0b00_11_11_00), + Shuffle::AlternateCB => do_shuffle!(0b01_10_01_10), + #[cfg(any(test, feature = "ed25519"))] + Shuffle::RepeatAB => do_shuffle!(0b01_00_01_00), } } @@ -285,8 +309,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn blend(&self, other: &Self, lanes: Lanes) -> Self { + pub(crate) fn blend(&self, other: &Self, lanes: Lanes) -> Self { // u64 lane → u32 pair: A={0,1}, B={2,3}, C={4,5}, D={6,7} macro_rules! do_blend { ($imm:expr) => { @@ -301,17 +324,17 @@ impl FieldElement51x4 { } match lanes { - Lanes::A => do_blend!(0b0000_0011), - Lanes::B => do_blend!(0b0000_1100), + #[cfg(any(test, feature = "ed25519"))] Lanes::C => do_blend!(0b0011_0000), Lanes::D => do_blend!(0b1100_0000), Lanes::AB => do_blend!(0b0000_1111), Lanes::AC => do_blend!(0b0011_0011), + #[cfg(any(test, feature = "ed25519"))] Lanes::AD => do_blend!(0b1100_0011), + #[cfg(any(test, feature = "ed25519"))] Lanes::BC => do_blend!(0b0011_1100), - Lanes::BCD => do_blend!(0b1111_1100), - Lanes::CD => do_blend!(0b1111_0000), - Lanes::ABCD => do_blend!(0b1111_1111), + #[cfg(any(test, feature = "ed25519"))] + Lanes::ExceptA => do_blend!(0b1111_1100), } } @@ -322,9 +345,8 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn select_mask(&self, other: &Self, mask: u64) -> Self { - let mask = _mm256_set1_epi64x(mask as i64); + pub(crate) fn select_mask(&self, other: &Self, mask: u64) -> Self { + let mask = _mm256_set1_epi64x(u64_as_i64_bits(mask)); Self([ _mm256_xor_si256( self.0[0], @@ -356,9 +378,8 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn diff_sum(&self) -> Self { - let swapped = self.shuffle(Shuffle::BADC); // (B, A, D, C) + pub(crate) fn diff_sum(&self) -> Self { + let swapped = self.shuffle(Shuffle::SwapPairs); // (B, A, D, C) let negated = self.negate_lazy(); // (-A, -B, -C, -D) let neg_ac = self.blend(&negated, Lanes::AC); // (-A, B, -C, D) swapped.add(&neg_ac) // (B-A, A+B, D-C, C+D) @@ -371,8 +392,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn reduce(mut self) -> Self { + pub(crate) fn reduce(mut self) -> Self { let mask = _mm256_set1_epi64x(MASK51); let r19 = _mm256_set1_epi64x(19); @@ -423,8 +443,7 @@ impl FieldElement51x4 { /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn mul(&self, rhs: &Self) -> Self { + pub(crate) fn mul(&self, rhs: &Self) -> Self { let zero = _mm256_setzero_si256(); let f = &self.0; let g = &rhs.0; @@ -579,8 +598,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX-512 IFMA + VL are available. #[cfg(test)] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn mul_unreduced(&self, rhs: &Self) -> Self { + pub(crate) fn mul_unreduced(&self, rhs: &Self) -> Self { let zero = _mm256_setzero_si256(); let mask52 = _mm256_set1_epi64x(MASK52); @@ -882,8 +900,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX-512 IFMA + VL are available. #[cfg(test)] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn mul_small_unreduced(&self, small: &Self) -> Self { + pub(crate) fn mul_small_unreduced(&self, small: &Self) -> Self { let zero = _mm256_setzero_si256(); let mask52 = _mm256_set1_epi64x(MASK52); let c = small.0[0]; // The only non-zero limb (≤18 bits, fits in 52 bits) @@ -950,8 +967,8 @@ impl FieldElement51x4 { /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn square(&self) -> Self { + #[cfg(any(test, feature = "ed25519"))] + pub(crate) fn square(&self) -> Self { let zero = _mm256_setzero_si256(); let f = &self.0; @@ -1056,8 +1073,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX-512 IFMA + VL are available. #[cfg(test)] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - unsafe fn square_wide_fold(&self) -> [__m256i; 5] { + fn square_wide_fold(&self) -> [__m256i; 5] { let zero = _mm256_setzero_si256(); let f = &self.0; @@ -1179,8 +1195,7 @@ impl FieldElement51x4 { #[cfg(test)] #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn square_and_negate_d_wide(&self) -> Self { + pub(crate) fn square_and_negate_d_wide(&self) -> Self { let mut folded = self.square_wide_fold(); // Negate D lane in u64 domain using p × 2^10 bias. @@ -1229,8 +1244,7 @@ impl FieldElement51x4 { /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn mul_small(&self, small: &Self) -> Self { + pub(crate) fn mul_small(&self, small: &Self) -> Self { let zero = _mm256_setzero_si256(); let f = &self.0; let c = small.0[0]; // The only non-zero limb @@ -1468,7 +1482,7 @@ mod tests { // SAFETY: AVX2 checked above. unsafe { let packed = FieldElement51x4::new(&a, &b, &c, &d); - let shuffled = packed.shuffle(Shuffle::BADC); + let shuffled = packed.shuffle(Shuffle::SwapPairs); let [ra, rb, rc, rd] = shuffled.split(); assert_eq!(ra.limbs(), b.limbs(), "BADC lane 0 = B"); diff --git a/src/auth/ed25519/point.rs b/src/auth/ed25519/point.rs index ec0c014f..cad1bfe1 100644 --- a/src/auth/ed25519/point.rs +++ b/src/auth/ed25519/point.rs @@ -5,16 +5,23 @@ use core::fmt; -use super::{field::FieldElement, scalar}; +#[cfg(feature = "ed25519")] +use super::scalar; +use super::{field::FieldElement, scalar_radix_16}; +#[cfg(feature = "ed25519")] use crate::traits::ct; #[path = "basepoint_tables.rs"] mod basepoint_tables; +#[cfg(feature = "ed25519")] #[path = "basepoint_wnaf5_table.rs"] mod basepoint_wnaf5_table; -pub(crate) use self::{basepoint_tables::BASEPOINT_RADIX16_TABLE, basepoint_wnaf5_table::BASEPOINT_WNAF5_TABLE}; +pub(crate) use self::basepoint_tables::BASEPOINT_RADIX16_TABLE; +#[cfg(feature = "ed25519")] +pub(crate) use self::basepoint_wnaf5_table::BASEPOINT_WNAF5_TABLE; +#[cfg(feature = "ed25519")] const EDWARDS_D: FieldElement = FieldElement::from_limbs([ 929_955_233_495_203, 466_365_720_129_213, @@ -22,6 +29,7 @@ const EDWARDS_D: FieldElement = FieldElement::from_limbs([ 2_033_849_074_728_123, 1_442_794_654_840_575, ]); +#[cfg(feature = "ed25519")] const EDWARDS_D2: FieldElement = FieldElement::from_limbs([ 1_859_910_466_990_425, 932_731_440_258_426, @@ -29,6 +37,7 @@ const EDWARDS_D2: FieldElement = FieldElement::from_limbs([ 1_815_898_335_770_999, 633_789_495_995_903, ]); +#[cfg(feature = "ed25519")] const BASEPOINT_X: FieldElement = FieldElement::from_limbs([ 1_738_742_601_995_546, 1_146_398_526_822_698, @@ -36,6 +45,7 @@ const BASEPOINT_X: FieldElement = FieldElement::from_limbs([ 562_264_141_797_630, 587_772_402_128_613, ]); +#[cfg(feature = "ed25519")] const BASEPOINT_Y: FieldElement = FieldElement::from_limbs([ 1_801_439_850_948_184, 1_351_079_888_211_148, @@ -95,6 +105,7 @@ impl CachedPoint { /// multiply per addition than affine `CachedPoint`, but avoids the /// expensive field inversion required to convert runtime tables to affine. #[derive(Clone, Copy)] +#[cfg(feature = "ed25519")] struct ProjectiveCachedPoint { y_plus_x: FieldElement, y_minus_x: FieldElement, @@ -102,6 +113,7 @@ struct ProjectiveCachedPoint { t2d: FieldElement, } +#[cfg(feature = "ed25519")] impl ProjectiveCachedPoint { const IDENTITY: Self = Self { y_plus_x: FieldElement::ONE, @@ -144,6 +156,7 @@ impl ExtendedPoint { } /// Construct an extended point from affine coordinates. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn from_affine(x: FieldElement, y: FieldElement) -> Self { Self { @@ -155,6 +168,7 @@ impl ExtendedPoint { } /// Add two extended Edwards points. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn add(&self, rhs: &Self) -> Self { let a = self.y.sub(&self.x).mul(&rhs.y.sub(&rhs.x)); @@ -197,6 +211,7 @@ impl ExtendedPoint { /// Add a projective cached point (8M — 1 more than affine cached, but /// avoids the inversion needed to build affine runtime tables). + #[cfg(feature = "ed25519")] #[must_use] fn add_projective_cached(&self, rhs: &ProjectiveCachedPoint) -> Self { let a = self.y.sub(&self.x).mul(&rhs.y_minus_x); @@ -223,6 +238,7 @@ impl ExtendedPoint { /// multiplications, no curve-constant multiply. The general `add(self)` /// path costs 4 squarings + 5 multiplications and can't exploit squaring /// symmetry in the compiler. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn double(&self) -> Self { let a = self.x.square(); @@ -244,6 +260,7 @@ impl ExtendedPoint { } /// Compress the point into the standard Ed25519 encoding. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn to_bytes(self) -> Option<[u8; 32]> { let (x, y) = self.to_affine()?; @@ -253,6 +270,7 @@ impl ExtendedPoint { } /// Decode a compressed Ed25519 point. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn from_bytes(bytes: &[u8; 32]) -> Option { let sign = (bytes[31] >> 7) != 0; @@ -275,12 +293,14 @@ impl ExtendedPoint { } /// Standard Ed25519 basepoint. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn basepoint() -> Self { Self::from_affine(BASEPOINT_X, BASEPOINT_Y) } /// Scalar multiplication by a little-endian 32-byte scalar. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn scalar_mul(&self, scalar: &[u8; 32]) -> Self { let mut acc = Self::identity(); @@ -301,9 +321,10 @@ impl ExtendedPoint { /// Variable-base signed radix-16 multiplication using a projective runtime /// table (no field inversion). + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn scalar_mul_vartime(&self, scalar: &[u8; 32]) -> Self { - let digits = scalar::as_radix_16(scalar); + let digits = scalar_radix_16(scalar); let table = projective_cached_multiples(self); let mut acc = Self::identity(); @@ -324,7 +345,7 @@ impl ExtendedPoint { /// repeated-doubling work from the fixed-base path. #[must_use] pub(crate) fn scalar_mul_basepoint(scalar: &[u8; 32]) -> Self { - let digits = scalar::as_radix_16(scalar); + let digits = scalar_radix_16(scalar); let mut acc = Self::identity(); for (digit, table) in digits.iter().copied().zip(BASEPOINT_RADIX16_TABLE.iter()) { @@ -338,18 +359,21 @@ impl ExtendedPoint { // Radix-16 Straus removed — superseded by straus_wnaf_basepoint_vartime. /// Multiply by the Edwards cofactor. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn mul_by_cofactor(&self) -> Self { self.double().double().double() } /// Whether this point lies in the low-order torsion subgroup. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn is_small_order(&self) -> bool { self.mul_by_cofactor().equals_projective(&Self::identity()) } /// Convert the point to affine coordinates when `Z != 0`. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn to_affine(self) -> Option<(FieldElement, FieldElement)> { if self.z.is_zero() { @@ -365,8 +389,22 @@ impl ExtendedPoint { /// `u = (Z + Y) / (Z - Y)` is the standard birational map from /// Edwards25519 to Curve25519. The identity maps to the Montgomery /// 2-torsion point `u = 0`. + #[cfg(all( + feature = "x25519", + any( + test, + miri, + not(any( + all( + target_arch = "aarch64", + any(target_os = "macos", target_os = "linux"), + not(feature = "portable-only") + ), + all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) + )) + ) + ))] #[must_use] - #[allow(dead_code)] pub(crate) fn to_montgomery_u(self) -> FieldElement { let numerator = self.z.add(&self.y); let denominator = self.z.sub(&self.y); @@ -378,6 +416,7 @@ impl ExtendedPoint { } /// Compare two extended points without converting to affine coordinates. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn equals_projective(&self, rhs: &Self) -> bool { let zero = [0u8; 32]; @@ -402,6 +441,7 @@ impl ExtendedPoint { } /// Borrow the extended-coordinate components. + #[cfg(any(feature = "ed25519", target_arch = "x86_64"))] #[must_use] pub(crate) const fn components(&self) -> (&FieldElement, &FieldElement, &FieldElement, &FieldElement) { (&self.x, &self.y, &self.z, &self.t) @@ -419,16 +459,27 @@ fn ct_eq_mask_u8(lhs: u8, rhs: u8) -> u64 { #[inline(always)] #[must_use] fn ct_negative_mask_i8(value: i8) -> u64 { - let bit = ((i16::from(value) >> 15) & 1) as u64; + let bit = u64::from(value.to_ne_bytes()[0] >> 7); 0u64.wrapping_sub(bit) } #[inline(always)] #[must_use] fn ct_abs_i8(value: i8) -> u8 { - let value = i16::from(value); - let sign = value >> 15; - ((value ^ sign) - sign) as u8 + let value = value.to_ne_bytes()[0]; + let sign = value >> 7; + let mask = 0u8.wrapping_sub(sign); + (value ^ mask).wrapping_add(sign) +} + +#[inline] +fn volatile_copy_field(field: &FieldElement) -> FieldElement { + let mut limbs = [0u64; 5]; + for (output, input) in limbs.iter_mut().zip(field.limbs()) { + // SAFETY: `input` is an aligned reference to an initialized `u64` that remains live for this read. + *output = unsafe { core::ptr::read_volatile(input) }; + } + FieldElement::from_limbs(limbs) } #[inline(always)] @@ -456,15 +507,13 @@ fn select_cached(lhs: &CachedPoint, rhs: &CachedPoint, mask: u64) -> CachedPoint fn select_signed_cached(table: &[CachedPoint; 8], digit: i8) -> CachedPoint { let abs = core::hint::black_box(ct_abs_i8(digit)); let mut selected = CachedPoint::IDENTITY; - for (i, candidate) in table.iter().enumerate() { - // SAFETY: Volatile table read is used as a compiler barrier because: - // 1. `candidate` is a valid shared reference to one `CachedPoint` entry. - // 2. `CachedPoint` is `Copy`, so the volatile read does not create ownership aliasing. - // 3. Every table entry is read unconditionally; the selected digit affects only masks below. - // Without this barrier, LLVM can rewrite the masked selection into a secret-dependent branch - // ladder. - let candidate = unsafe { core::ptr::read_volatile(candidate) }; - let mask = core::hint::black_box(ct_eq_mask_u8(abs, (i as u8).wrapping_add(1))); + for (expected, candidate) in (1u8..=8).zip(table) { + let candidate = CachedPoint { + y_plus_x: volatile_copy_field(&candidate.y_plus_x), + y_minus_x: volatile_copy_field(&candidate.y_minus_x), + t2d: volatile_copy_field(&candidate.t2d), + }; + let mask = core::hint::black_box(ct_eq_mask_u8(abs, expected)); selected = select_cached(&selected, &candidate, mask); } @@ -472,6 +521,7 @@ fn select_signed_cached(table: &[CachedPoint; 8], digit: i8) -> CachedPoint { select_cached(&selected, &neg, core::hint::black_box(ct_negative_mask_i8(digit))) } +/// Select one signed digit from the first portable basepoint table and return its field limbs. #[cfg(feature = "diag")] #[inline(always)] pub fn diag_select_basepoint_cached_limb_digest(digit: i8) -> [u64; 15] { @@ -484,6 +534,7 @@ pub fn diag_select_basepoint_cached_limb_digest(digit: i8) -> [u64; 15] { } /// Add a signed digit from a projective cached table (runtime table). +#[cfg(feature = "ed25519")] #[inline] #[must_use] fn add_signed_projective_cached(acc: ExtendedPoint, table: &[ProjectiveCachedPoint; 8], digit: i8) -> ExtendedPoint { @@ -502,6 +553,7 @@ fn add_signed_projective_cached(acc: ExtendedPoint, table: &[ProjectiveCachedPoi /// Build a projective cached table of `[1P, 2P, ..., 8P]` without any /// field inversion. Each entry stores `(Y+X, Y-X, Z, 2dT)` in extended /// projective coordinates. +#[cfg(feature = "ed25519")] #[must_use] fn projective_cached_multiples(point: &ExtendedPoint) -> [ProjectiveCachedPoint; 8] { let mut out = [ProjectiveCachedPoint::IDENTITY; 8]; @@ -517,6 +569,7 @@ fn projective_cached_multiples(point: &ExtendedPoint) -> [ProjectiveCachedPoint; } /// Build odd multiples `[1P, 3P, 5P, ..., (2n-1)P]` in projective cached format. +#[cfg(feature = "ed25519")] #[must_use] fn odd_projective_cached_multiples(point: &ExtendedPoint) -> [ProjectiveCachedPoint; N] { let p2 = point.double(); @@ -533,6 +586,7 @@ fn odd_projective_cached_multiples(point: &ExtendedPoint) -> [Pr } /// Add a signed wNAF digit from an odd-multiples projective cached table. +#[cfg(feature = "ed25519")] #[inline] #[must_use] fn add_wnaf_digit_projective(acc: ExtendedPoint, table: &[ProjectiveCachedPoint], digit: i8) -> ExtendedPoint { @@ -549,6 +603,7 @@ fn add_wnaf_digit_projective(acc: ExtendedPoint, table: &[ProjectiveCachedPoint] } /// Add a signed wNAF digit from an affine cached table. +#[cfg(feature = "ed25519")] #[inline] #[must_use] fn add_wnaf_digit_cached(acc: ExtendedPoint, table: &[CachedPoint], digit: i8) -> ExtendedPoint { @@ -567,8 +622,8 @@ fn add_wnaf_digit_cached(acc: ExtendedPoint, table: &[CachedPoint], digit: i8) - /// wNAF-based portable Straus: `[s]B + [h]A`. /// /// Uses wNAF(5) for both scalars (8-entry odd-multiples tables). +#[cfg(feature = "ed25519")] #[must_use] -#[allow(clippy::indexing_slicing)] // i bounded by top < 256, naf arrays are [i8; 256] pub(crate) fn straus_wnaf_basepoint_vartime(s: &[u8; 32], h: &[u8; 32], a: &ExtendedPoint) -> ExtendedPoint { let s_naf = scalar::non_adjacent_form(s, 5); let h_naf = scalar::non_adjacent_form(h, 5); @@ -617,75 +672,18 @@ impl fmt::Debug for ExtendedPoint { } } -#[cfg(test)] +#[cfg(all(test, feature = "ed25519"))] mod tests { use super::{ExtendedPoint, FieldElement}; - /// Generates the wNAF(8) basepoint table for IFMA verify. - /// Run with `--nocapture` to print the table source. - #[cfg(feature = "std")] - #[test] - fn gen_basepoint_wnaf8_ifma_table() { - let bp = ExtendedPoint::basepoint(); - let bp2 = bp.double(); - - let d2_fe = FieldElement::from_small(121_666); - let d2_2_fe = FieldElement::from_small(121_666u64.wrapping_mul(2)); - let d1_2_fe = FieldElement::from_small(121_665u64.wrapping_mul(2)); - - let mut acc = bp; - eprintln!("pub(crate) static BASEPOINT_WNAF8_IFMA_RAW: [[[i64; 4]; 5]; 64] = ["); - for i in 0..64u32 { - if i > 0 { - acc = acc.add(&bp2); - } - let (x, y, z, t) = acc.components(); - - let a = d2_fe.mul(&y.sub(x)).normalize(); - let b = d2_fe.mul(&y.add(x)).normalize(); - let c = d2_2_fe.mul(z).normalize(); - let d = d1_2_fe.mul(t).neg().normalize(); - - let al = a.limbs(); - let bl = b.limbs(); - let cl = c.limbs(); - let dl = d.limbs(); - - eprintln!(" // {}B", 2 * i + 1); - eprintln!(" ["); - for k in 0..5 { - eprintln!( - " [{}, {}, {}, {}],", - al[k] as i64, bl[k] as i64, cl[k] as i64, dl[k] as i64 - ); - } - eprintln!(" ],"); - } - eprintln!("];"); - } - fn basepoint() -> ExtendedPoint { ExtendedPoint::basepoint() } - fn decode_hex_32(hex: &str) -> [u8; 32] { - let bytes = hex.as_bytes(); + fn decode_hex_32(hex: &str) -> Result<[u8; 32], crate::hex::InvalidHexError> { let mut out = [0u8; 32]; - - for (dst, chunk) in out.iter_mut().zip(bytes.chunks_exact(2)) { - *dst = hex_value(chunk[0]) << 4 | hex_value(chunk[1]); - } - - out - } - - fn hex_value(byte: u8) -> u8 { - match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex"), - } + crate::hex::from_hex(hex, &mut out)?; + Ok(out) } #[test] @@ -722,15 +720,15 @@ mod tests { #[test] fn basepoint_roundtrips_compressed_encoding() { - let expected = decode_hex_32("5866666666666666666666666666666666666666666666666666666666666666"); + let expected = decode_hex_32("5866666666666666666666666666666666666666666666666666666666666666").ok(); let encoded = basepoint().to_bytes(); - assert_eq!(encoded, Some(expected)); + assert_eq!(encoded, expected); assert_eq!( encoded .and_then(|bytes| ExtendedPoint::from_bytes(&bytes)) .and_then(|point| point.to_bytes()), - Some(expected) + expected ); } @@ -781,8 +779,7 @@ mod tests { #[test] fn basepoint_wnaf5_table_matches_odd_basepoint_multiples() { - for (i, cached) in super::BASEPOINT_WNAF5_TABLE.iter().enumerate() { - let scalar = (2 * i + 1) as u8; + for (scalar, cached) in (1u8..).step_by(2).zip(&super::BASEPOINT_WNAF5_TABLE) { let mut scalar_bytes = [0u8; 32]; scalar_bytes[0] = scalar; @@ -808,13 +805,13 @@ mod tests { fn rfc8032_public_key_derivation_matches_vector_1() { use crate::auth::ed25519::{Ed25519SecretKey, hash::ExpandedSecret}; - let secret = Ed25519SecretKey::from_bytes(decode_hex_32( - "9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60", - )); + let secret_bytes = decode_hex_32("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60").ok(); + assert!(secret_bytes.is_some()); + let secret = Ed25519SecretKey::from_bytes(secret_bytes.unwrap_or_default()); let expanded = ExpandedSecret::from_secret_key(&secret); let public = ExtendedPoint::scalar_mul_basepoint(expanded.scalar_bytes()).to_bytes(); - let expected = decode_hex_32("d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a"); + let expected = decode_hex_32("d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a").ok(); - assert_eq!(public, Some(expected)); + assert_eq!(public, expected); } } diff --git a/src/auth/ed25519/point_avx2.rs b/src/auth/ed25519/point_avx2.rs index 2eb34880..a15b6fa4 100644 --- a/src/auth/ed25519/point_avx2.rs +++ b/src/auth/ed25519/point_avx2.rs @@ -12,19 +12,22 @@ //! scaling all output coordinates by `d2²` — which cancels in projective //! coordinates. -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", any(test, feature = "ed25519")))] use core::arch::x86_64::_mm256_loadu_si256; +#[cfg(all(target_arch = "x86_64", feature = "ed25519"))] +use super::scalar; #[cfg(target_arch = "x86_64")] use super::{ field::FieldElement, field_avx2::{FieldElement2625x4, Lanes, Shuffle}, field_ifma::FieldElement51x4, point::{CachedPoint, ExtendedPoint}, - scalar, + scalar_radix_16, }; #[cfg(target_arch = "x86_64")] #[path = "basepoint_table_ifma.rs"] +#[cfg(any(test, feature = "ed25519"))] mod basepoint_table_ifma; /// Hamburg constants for the curve `d = -d1/d2`. @@ -57,8 +60,7 @@ impl ExtendedPointAvx2 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn from_extended(p: &ExtendedPoint) -> Self { + pub(crate) fn from_extended(p: &ExtendedPoint) -> Self { let (x, y, z, t) = p.components(); Self(FieldElement2625x4::new(x, y, z, t)) } @@ -70,8 +72,7 @@ impl ExtendedPointAvx2 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn to_extended(self) -> ExtendedPoint { + pub(crate) fn to_extended(self) -> ExtendedPoint { let [x, y, z, t] = self.0.split(); ExtendedPoint::from_raw(x, y, z, t) } @@ -85,8 +86,7 @@ impl ExtendedPointAvx2 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn to_cached(self) -> CachedPointAvx2 { + pub(crate) fn to_cached(self) -> CachedPointAvx2 { // Step 1: Compute (Y-X, Y+X) in lanes A,B; keep Z,T in C,D. let ds = self.0.diff_sum(); // (Y-X, Y+X, T-Z, Z+T) let prepared = self.0.blend(&ds, Lanes::AB); // (Y-X, Y+X, Z, T) @@ -109,8 +109,7 @@ impl ExtendedPointAvx2 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn add_cached(&self, other: &CachedPointAvx2) -> Self { + pub(crate) fn add_cached(&self, other: &CachedPointAvx2) -> Self { // Step 1: Prepare self as (Y-X, Y+X, Z, T) by blending diff_sum into A,B. let ds = self.0.diff_sum(); let tmp = self.0.blend(&ds, Lanes::AB); // (Y1-X1, Y1+X1, Z1, T1) @@ -121,14 +120,14 @@ impl ExtendedPointAvx2 { // Step 3: Swap C↔D to align for diff_sum. // After swap: lane C has the T-product, lane D has the Z-product. - let swapped = product.shuffle(Shuffle::ABDC); + let swapped = product.shuffle(Shuffle::SwapCD); // Step 4: diff_sum computes (e, h, f, g) (up to Hamburg scaling). let ehfg = swapped.diff_sum(); // Step 5: Shuffle into final multiply operands. - let t0 = ehfg.shuffle(Shuffle::ADDA); // (e, g, g, e) - let t1 = ehfg.shuffle(Shuffle::CBCB); // (f, h, f, h) + let t0 = ehfg.shuffle(Shuffle::OuterAInnerD); // (e, g, g, e) + let t1 = ehfg.shuffle(Shuffle::AlternateCB); // (f, h, f, h) // Step 6: Uniform multiply → (e·f, g·h, g·f, e·h) = (X3, Y3, Z3, T3). Self(t0.mul(&t1)) @@ -143,11 +142,11 @@ impl ExtendedPointAvx2 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn double(&self) -> Self { + #[cfg(any(test, feature = "ed25519"))] + pub(crate) fn double(&self) -> Self { // Step 1: Build (X, Y, Z, X+Y) for squaring. - let ab = self.0.shuffle(Shuffle::ABAB); // (X, Y, X, Y) - let ba = ab.shuffle(Shuffle::BADC); // (Y, X, Y, X) + let ab = self.0.shuffle(Shuffle::RepeatAB); // (X, Y, X, Y) + let ba = ab.shuffle(Shuffle::SwapPairs); // (Y, X, Y, X) let xy_sum = ab.add(&ba); // (X+Y, Y+X, X+Y, Y+X) let prepared = self.0.blend(&xy_sum, Lanes::D); // (X, Y, Z, X+Y) @@ -163,8 +162,8 @@ impl ExtendedPointAvx2 { // Double-negations cancel in the final multiply. let zero = FieldElement2625x4::zero(); - let s1 = sq.shuffle(Shuffle::AAAA); // (S1, S1, S1, S1) - let s2 = sq.shuffle(Shuffle::BBBB); // (S2, S2, S2, S2) + let s1 = sq.shuffle(Shuffle::BroadcastA); // (S1, S1, S1, S1) + let s2 = sq.shuffle(Shuffle::BroadcastB); // (S2, S2, S2, S2) // Build the target vector incrementally: let sq_doubled = sq.add(&sq); // (2S1, 2S2, 2S3, −2S4) @@ -186,8 +185,8 @@ impl ExtendedPointAvx2 { let tmp = tmp.add(&neg_s2_in_bc); // (S5, S6, S8, S9) // Step 4: Shuffle into final multiply operands. - let t0 = tmp.shuffle(Shuffle::CACA); // (S8, S5, S8, S5) - let t1 = tmp.shuffle(Shuffle::DBBD); // (S9, S6, S6, S9) + let t0 = tmp.shuffle(Shuffle::AlternateCA); // (S8, S5, S8, S5) + let t1 = tmp.shuffle(Shuffle::OuterDInnerB); // (S9, S6, S6, S9) // Step 5: Uniform multiply → (S8·S9, S5·S6, S8·S6, S5·S9) = (X3, Y3, Z3, T3). Self(t0.mul(&t1)) @@ -206,9 +205,8 @@ impl CachedPointAvx2 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn neg(&self) -> Self { - let swapped = self.0.shuffle(Shuffle::BACD); // swap A↔B, keep C,D + pub(crate) fn neg(&self) -> Self { + let swapped = self.0.shuffle(Shuffle::SwapAB); // swap A↔B, keep C,D let negated = swapped.negate_lazy(); Self(swapped.blend(&negated, Lanes::D)) // negate D only } @@ -225,16 +223,27 @@ fn ct_eq_mask_u8(lhs: u8, rhs: u8) -> u64 { #[inline(always)] #[must_use] fn ct_negative_mask_i8(value: i8) -> u64 { - let bit = ((i16::from(value) >> 15) & 1) as u64; + let bit = u64::from(value.to_ne_bytes()[0] >> 7); 0u64.wrapping_sub(bit) } #[inline(always)] #[must_use] fn ct_abs_i8(value: i8) -> u8 { - let value = i16::from(value); - let sign = value >> 15; - ((value ^ sign) - sign) as u8 + let value = value.to_ne_bytes()[0]; + let sign = value >> 7; + let mask = 0u8.wrapping_sub(sign); + (value ^ mask).wrapping_add(sign) +} + +#[inline] +fn volatile_copy_field(field: &FieldElement) -> FieldElement { + let mut limbs = [0u64; 5]; + for (output, input) in limbs.iter_mut().zip(field.limbs()) { + // SAFETY: `input` is an aligned reference to an initialized `u64` that remains live for this read. + *output = unsafe { core::ptr::read_volatile(input) }; + } + FieldElement::from_limbs(limbs) } /// Fixed-schedule cached-point selection for AVX2 fixed-base tables. @@ -244,8 +253,7 @@ fn ct_abs_i8(value: i8) -> u8 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn select_cached_avx2(lhs: &CachedPointAvx2, rhs: &CachedPointAvx2, mask: u64) -> CachedPointAvx2 { +fn select_cached_avx2(lhs: &CachedPointAvx2, rhs: &CachedPointAvx2, mask: u64) -> CachedPointAvx2 { CachedPointAvx2(lhs.0.select_mask(&rhs.0, mask)) } @@ -256,8 +264,7 @@ unsafe fn select_cached_avx2(lhs: &CachedPointAvx2, rhs: &CachedPointAvx2, mask: /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn hamburg_constants() -> FieldElement2625x4 { +fn hamburg_constants() -> FieldElement2625x4 { let d2_fe = FieldElement::from_small(D2); let d2_fe_2 = FieldElement::from_small(D2.wrapping_mul(2)); let d1_fe_2 = FieldElement::from_small(D1.wrapping_mul(2)); @@ -274,8 +281,7 @@ unsafe fn hamburg_constants() -> FieldElement2625x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn hamburg_affine_constants() -> FieldElement2625x4 { +fn hamburg_affine_constants() -> FieldElement2625x4 { let d2_fe = FieldElement::from_small(D2); let d2_fe_2 = FieldElement::from_small(D2.wrapping_mul(2)); FieldElement2625x4::new(&d2_fe, &d2_fe, &d2_fe_2, &d2_fe) @@ -298,8 +304,8 @@ unsafe fn hamburg_affine_constants() -> FieldElement2625x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn cached_from_affine(cp: &CachedPoint, constants: &FieldElement2625x4) -> CachedPointAvx2 { +#[cfg(any(test, feature = "ed25519"))] +fn cached_from_affine(cp: &CachedPoint, constants: &FieldElement2625x4) -> CachedPointAvx2 { let (y_plus_x, y_minus_x, t2d) = cp.components(); let packed = FieldElement2625x4::new(y_minus_x, y_plus_x, &FieldElement::ONE, t2d); CachedPointAvx2(packed.mul(constants)) @@ -312,8 +318,7 @@ unsafe fn cached_from_affine(cp: &CachedPoint, constants: &FieldElement2625x4) - /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn select_signed_cached_avx2( +fn select_signed_cached_avx2( table: &[CachedPoint; 8], digit: i8, affine_k: &FieldElement2625x4, @@ -321,10 +326,14 @@ unsafe fn select_signed_cached_avx2( ) -> CachedPointAvx2 { let abs = core::hint::black_box(ct_abs_i8(digit)); let mut selected = *identity; - for (i, candidate) in table.iter().enumerate() { - let candidate = core::ptr::read_volatile(candidate); - let cached = cached_from_affine(&candidate, affine_k); - let mask = core::hint::black_box(ct_eq_mask_u8(abs, (i as u8).wrapping_add(1))); + for (expected, candidate) in (1u8..=8).zip(table) { + let (y_plus_x, y_minus_x, t2d) = candidate.components(); + let y_plus_x = volatile_copy_field(y_plus_x); + let y_minus_x = volatile_copy_field(y_minus_x); + let t2d = volatile_copy_field(t2d); + let packed = FieldElement2625x4::new(&y_minus_x, &y_plus_x, &FieldElement::ONE, &t2d); + let cached = CachedPointAvx2(packed.mul(affine_k)); + let mask = core::hint::black_box(ct_eq_mask_u8(abs, expected)); selected = select_cached_avx2(&selected, &cached, mask); } @@ -339,8 +348,8 @@ unsafe fn select_signed_cached_avx2( /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add_wnaf_digit_cached_avx2( +#[cfg(any(test, feature = "ed25519"))] +fn add_wnaf_digit_cached_avx2( acc: ExtendedPointAvx2, table: &[CachedPoint; 8], digit: i8, @@ -366,8 +375,8 @@ unsafe fn add_wnaf_digit_cached_avx2( /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add_signed_runtime_cached_avx2( +#[cfg(any(test, feature = "ed25519"))] +fn add_signed_runtime_cached_avx2( acc: ExtendedPointAvx2, table: &[CachedPointAvx2; 8], digit: i8, @@ -390,8 +399,8 @@ unsafe fn add_signed_runtime_cached_avx2( /// /// Caller must ensure AVX2 is available. #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn cached_multiples_avx2(point: &ExtendedPointAvx2) -> [CachedPointAvx2; 8] { +#[cfg(any(test, feature = "ed25519"))] +fn cached_multiples_avx2(point: &ExtendedPointAvx2) -> [CachedPointAvx2; 8] { let mut acc = *point; let point_cached = point.to_cached(); let first = acc.to_cached(); @@ -415,9 +424,9 @@ unsafe fn cached_multiples_avx2(point: &ExtendedPointAvx2) -> [CachedPointAvx2; /// /// Caller must ensure AVX2 is available. #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] +#[cfg(feature = "ed25519")] pub(crate) unsafe fn scalar_mul_vartime_avx2(point: &ExtendedPoint, scalar_bytes: &[u8; 32]) -> ExtendedPoint { - let digits = scalar::as_radix_16(scalar_bytes); + let digits = scalar_radix_16(scalar_bytes); let avx_point = ExtendedPointAvx2::from_extended(point); let table = cached_multiples_avx2(&avx_point); @@ -442,11 +451,10 @@ pub(crate) unsafe fn scalar_mul_vartime_avx2(point: &ExtendedPoint, scalar_bytes /// /// Caller must ensure AVX2 is available. #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] pub(crate) unsafe fn scalar_mul_basepoint_avx2(scalar_bytes: &[u8; 32]) -> ExtendedPoint { use super::point::BASEPOINT_RADIX16_TABLE; - let digits = scalar::as_radix_16(scalar_bytes); + let digits = scalar_radix_16(scalar_bytes); let affine_k = hamburg_affine_constants(); let mut acc = ExtendedPointAvx2::from_extended(&ExtendedPoint::identity()); let identity = acc.to_cached(); @@ -467,7 +475,6 @@ pub(crate) unsafe fn scalar_mul_basepoint_avx2(scalar_bytes: &[u8; 32]) -> Exten #[cfg(feature = "diag")] #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] pub unsafe fn diag_select_basepoint_cached_avx2_limb_digest(digit: i8) -> [u64; 20] { use super::point::BASEPOINT_RADIX16_TABLE; @@ -491,8 +498,8 @@ pub unsafe fn diag_select_basepoint_cached_avx2_limb_digest(digit: i8) -> [u64; /// /// Caller must ensure AVX2 is available. #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn odd_multiples_avx2(point: &ExtendedPointAvx2) -> [CachedPointAvx2; N] { +#[cfg(any(test, feature = "ed25519"))] +fn odd_multiples_avx2(point: &ExtendedPointAvx2) -> [CachedPointAvx2; N] { let p2 = point.double(); let p2_cached = p2.to_cached(); @@ -513,8 +520,8 @@ unsafe fn odd_multiples_avx2(point: &ExtendedPointAvx2) -> [Cach /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add_wnaf_digit_avx2(acc: ExtendedPointAvx2, table: &[CachedPointAvx2], digit: i8) -> ExtendedPointAvx2 { +#[cfg(any(test, feature = "ed25519"))] +fn add_wnaf_digit_avx2(acc: ExtendedPointAvx2, table: &[CachedPointAvx2], digit: i8) -> ExtendedPointAvx2 { let index = usize::from((digit.unsigned_abs().wrapping_sub(1)) / 2); let Some(point) = table.get(index) else { return acc; @@ -540,8 +547,7 @@ unsafe fn add_wnaf_digit_avx2(acc: ExtendedPointAvx2, table: &[CachedPointAvx2], /// /// Caller must ensure AVX2 is available. #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -#[allow(clippy::indexing_slicing)] // i bounded by top < 256, naf arrays are [i8; 256] +#[cfg(feature = "ed25519")] pub(crate) unsafe fn straus_wnaf_vartime_avx2(s: &[u8; 32], h: &[u8; 32], a: &ExtendedPoint) -> ExtendedPoint { use super::point::BASEPOINT_WNAF5_TABLE; @@ -608,8 +614,7 @@ impl ExtendedPointIfma { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn from_extended(p: &ExtendedPoint) -> Self { + pub(crate) fn from_extended(p: &ExtendedPoint) -> Self { let (x, y, z, t) = p.components(); Self(FieldElement51x4::new(x, y, z, t)) } @@ -621,8 +626,7 @@ impl ExtendedPointIfma { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn to_extended(self) -> ExtendedPoint { + pub(crate) fn to_extended(self) -> ExtendedPoint { let [x, y, z, t] = self.0.split(); ExtendedPoint::from_raw(x, y, z, t) } @@ -634,8 +638,7 @@ impl ExtendedPointIfma { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn to_cached(self) -> CachedPointIfma { + pub(crate) fn to_cached(self) -> CachedPointIfma { let ds = self.0.diff_sum(); let prepared = self.0.blend(&ds, Lanes::AB); let constants = hamburg_constants_ifma(); @@ -651,16 +654,15 @@ impl ExtendedPointIfma { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn add_cached(&self, other: &CachedPointIfma) -> Self { + pub(crate) fn add_cached(&self, other: &CachedPointIfma) -> Self { let ds = self.0.diff_sum(); let tmp = self.0.blend(&ds, Lanes::AB); let product = tmp.reduce().mul(&other.0); - let swapped = product.shuffle(Shuffle::ABDC); + let swapped = product.shuffle(Shuffle::SwapCD); let ehfg = swapped.diff_sum(); let reduced = ehfg.reduce(); - let t0 = reduced.shuffle(Shuffle::ADDA); - let t1 = reduced.shuffle(Shuffle::CBCB); + let t0 = reduced.shuffle(Shuffle::OuterAInnerD); + let t1 = reduced.shuffle(Shuffle::AlternateCB); Self(t0.mul(&t1)) } @@ -674,11 +676,11 @@ impl ExtendedPointIfma { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn double(&self) -> Self { + #[cfg(any(test, feature = "ed25519"))] + pub(crate) fn double(&self) -> Self { // Prepare (X, Y, Z, X+Y) for squaring. - let tmp0 = self.0.shuffle(Shuffle::BADC); // (Y, X, _, _) - let tmp1 = self.0.add(&tmp0).shuffle(Shuffle::ABAB); // (X+Y, X+Y, X+Y, X+Y) + let tmp0 = self.0.shuffle(Shuffle::SwapPairs); // (Y, X, _, _) + let tmp1 = self.0.add(&tmp0).shuffle(Shuffle::RepeatAB); // (X+Y, X+Y, X+Y, X+Y) let prepared = self.0.blend(&tmp1, Lanes::D); // (X, Y, Z, X+Y) // Square reduced inputs → (S1, S2, S3, S4) = (X², Y², Z², (X+Y)²) @@ -688,8 +690,8 @@ impl ExtendedPointIfma { // Compute (S5, S6, S8, S9) where: // S5 = S1+S2, S6 = S1-S2, S8 = S1-S2+2S3, S9 = S1+S2-S4 let zero = FieldElement51x4::zero(); - let s1 = sq.shuffle(Shuffle::AAAA); - let s2 = sq.shuffle(Shuffle::BBBB); + let s1 = sq.shuffle(Shuffle::BroadcastA); + let s2 = sq.shuffle(Shuffle::BroadcastB); // (-S2, -S2, -S2, -S4): negate S2 in A,B,C and S4 in D. let s2_s2_s2_s4 = s2.blend(&sq, Lanes::D).negate_lazy(); @@ -698,13 +700,13 @@ impl ExtendedPointIfma { // = (S1, S1, S1+2S3, S1) tmp0 = tmp0.add(&zero.blend(&s2, Lanes::AD)); // = (S1+S2, S1, S1+2S3, S1+S2) - tmp0 = tmp0.add(&zero.blend(&s2_s2_s2_s4, Lanes::BCD)); + tmp0 = tmp0.add(&zero.blend(&s2_s2_s2_s4, Lanes::ExceptA)); // = (S1+S2, S1-S2, S1+2S3-S2, S1+S2-S4) = (S5, S6, S8, S9) // Reduce before final multiply. let reduced = tmp0.reduce(); - let t0 = reduced.shuffle(Shuffle::CACA); // (S8, S5, S8, S5) - let t1 = reduced.shuffle(Shuffle::DBBD); // (S9, S6, S6, S9) + let t0 = reduced.shuffle(Shuffle::AlternateCA); // (S8, S5, S8, S5) + let t1 = reduced.shuffle(Shuffle::OuterDInnerB); // (S9, S6, S6, S9) // (S8·S9, S5·S6, S8·S6, S5·S9) = (X3, Y3, Z3, T3) Self(t0.mul(&t1)) @@ -720,9 +722,8 @@ impl CachedPointIfma { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn neg(&self) -> Self { - let swapped = self.0.shuffle(Shuffle::BACD); + pub(crate) fn neg(&self) -> Self { + let swapped = self.0.shuffle(Shuffle::SwapAB); let negated = swapped.negate_lazy(); Self(swapped.blend(&negated, Lanes::D)) } @@ -735,8 +736,7 @@ impl CachedPointIfma { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn select_cached_ifma(lhs: &CachedPointIfma, rhs: &CachedPointIfma, mask: u64) -> CachedPointIfma { +fn select_cached_ifma(lhs: &CachedPointIfma, rhs: &CachedPointIfma, mask: u64) -> CachedPointIfma { CachedPointIfma(lhs.0.select_mask(&rhs.0, mask)) } @@ -747,8 +747,7 @@ unsafe fn select_cached_ifma(lhs: &CachedPointIfma, rhs: &CachedPointIfma, mask: /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn hamburg_constants_ifma() -> FieldElement51x4 { +fn hamburg_constants_ifma() -> FieldElement51x4 { let d2_fe = FieldElement::from_small(D2); let d2_fe_2 = FieldElement::from_small(D2.wrapping_mul(2)); let d1_fe_2 = FieldElement::from_small(D1.wrapping_mul(2)); @@ -762,29 +761,12 @@ unsafe fn hamburg_constants_ifma() -> FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn hamburg_affine_constants_ifma() -> FieldElement51x4 { +fn hamburg_affine_constants_ifma() -> FieldElement51x4 { let d2_fe = FieldElement::from_small(D2); let d2_fe_2 = FieldElement::from_small(D2.wrapping_mul(2)); FieldElement51x4::new(&d2_fe, &d2_fe, &d2_fe_2, &d2_fe) } -/// Convert an affine `CachedPoint` to Hamburg-scaled IFMA cached format. -/// -/// # Safety -/// -/// Caller must ensure AVX-512 IFMA + VL are available. -#[inline] -#[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn cached_from_affine_ifma(cp: &CachedPoint, constants: &FieldElement51x4) -> CachedPointIfma { - let (y_plus_x, y_minus_x, t2d) = cp.components(); - let packed = FieldElement51x4::new(y_minus_x, y_plus_x, &FieldElement::ONE, t2d); - // Affine table entries are already reduced (≤51-bit limbs from static data). - CachedPointIfma(packed.mul_small(constants).reduce()) -} - -/// Add a signed digit from an affine cached table (IFMA). /// Select a signed digit from an affine cached basepoint table. /// /// # Safety @@ -792,8 +774,7 @@ unsafe fn cached_from_affine_ifma(cp: &CachedPoint, constants: &FieldElement51x4 /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn select_signed_cached_ifma( +fn select_signed_cached_ifma( table: &[CachedPoint; 8], digit: i8, affine_k: &FieldElement51x4, @@ -801,10 +782,14 @@ unsafe fn select_signed_cached_ifma( ) -> CachedPointIfma { let abs = core::hint::black_box(ct_abs_i8(digit)); let mut selected = *identity; - for (i, candidate) in table.iter().enumerate() { - let candidate = core::ptr::read_volatile(candidate); - let cached = cached_from_affine_ifma(&candidate, affine_k); - let mask = core::hint::black_box(ct_eq_mask_u8(abs, (i as u8).wrapping_add(1))); + for (expected, candidate) in (1u8..=8).zip(table) { + let (y_plus_x, y_minus_x, t2d) = candidate.components(); + let y_plus_x = volatile_copy_field(y_plus_x); + let y_minus_x = volatile_copy_field(y_minus_x); + let t2d = volatile_copy_field(t2d); + let packed = FieldElement51x4::new(&y_minus_x, &y_plus_x, &FieldElement::ONE, &t2d); + let cached = CachedPointIfma(packed.mul_small(affine_k).reduce()); + let mask = core::hint::black_box(ct_eq_mask_u8(abs, expected)); selected = select_cached_ifma(&selected, &cached, mask); } @@ -819,8 +804,8 @@ unsafe fn select_signed_cached_ifma( /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add_signed_runtime_cached_ifma( +#[cfg(any(test, feature = "ed25519"))] +fn add_signed_runtime_cached_ifma( acc: ExtendedPointIfma, table: &[CachedPointIfma; 8], digit: i8, @@ -842,8 +827,8 @@ unsafe fn add_signed_runtime_cached_ifma( /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn cached_multiples_ifma(point: &ExtendedPointIfma) -> [CachedPointIfma; 8] { +#[cfg(any(test, feature = "ed25519"))] +fn cached_multiples_ifma(point: &ExtendedPointIfma) -> [CachedPointIfma; 8] { let mut acc = *point; let point_cached = point.to_cached(); let first = acc.to_cached(); @@ -862,9 +847,9 @@ unsafe fn cached_multiples_ifma(point: &ExtendedPointIfma) -> [CachedPointIfma; /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] +#[cfg(feature = "ed25519")] pub(crate) unsafe fn scalar_mul_vartime_ifma(point: &ExtendedPoint, scalar_bytes: &[u8; 32]) -> ExtendedPoint { - let digits = scalar::as_radix_16(scalar_bytes); + let digits = scalar_radix_16(scalar_bytes); let ifma_point = ExtendedPointIfma::from_extended(point); let table = cached_multiples_ifma(&ifma_point); let mut acc = ExtendedPointIfma::from_extended(&ExtendedPoint::identity()); @@ -883,11 +868,10 @@ pub(crate) unsafe fn scalar_mul_vartime_ifma(point: &ExtendedPoint, scalar_bytes /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] pub(crate) unsafe fn scalar_mul_basepoint_ifma(scalar_bytes: &[u8; 32]) -> ExtendedPoint { use super::point::BASEPOINT_RADIX16_TABLE; - let digits = scalar::as_radix_16(scalar_bytes); + let digits = scalar_radix_16(scalar_bytes); let affine_k = hamburg_affine_constants_ifma(); let mut acc = ExtendedPointIfma::from_extended(&ExtendedPoint::identity()); let identity = acc.to_cached(); @@ -908,7 +892,6 @@ pub(crate) unsafe fn scalar_mul_basepoint_ifma(scalar_bytes: &[u8; 32]) -> Exten #[cfg(feature = "diag")] #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] pub unsafe fn diag_select_basepoint_cached_ifma_limb_digest(digit: i8) -> [u64; 20] { use super::point::BASEPOINT_RADIX16_TABLE; @@ -939,8 +922,8 @@ pub unsafe fn diag_select_basepoint_cached_ifma_limb_digest(digit: i8) -> [u64; /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn odd_multiples_ifma(point: &ExtendedPointIfma) -> [CachedPointIfma; N] { +#[cfg(any(test, feature = "ed25519"))] +fn odd_multiples_ifma(point: &ExtendedPointIfma) -> [CachedPointIfma; N] { let p2 = point.double(); let p2_cached = p2.to_cached(); @@ -964,8 +947,8 @@ unsafe fn odd_multiples_ifma(point: &ExtendedPointIfma) -> [Cach /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add_wnaf_digit_ifma(acc: ExtendedPointIfma, table: &[CachedPointIfma], digit: i8) -> ExtendedPointIfma { +#[cfg(any(test, feature = "ed25519"))] +fn add_wnaf_digit_ifma(acc: ExtendedPointIfma, table: &[CachedPointIfma], digit: i8) -> ExtendedPointIfma { let index = usize::from((digit.unsigned_abs().wrapping_sub(1)) / 2); let Some(point) = table.get(index) else { return acc; @@ -985,15 +968,19 @@ unsafe fn add_wnaf_digit_ifma(acc: ExtendedPointIfma, table: &[CachedPointIfma], /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn load_cached_ifma_raw(entry: &[[i64; 4]; 5]) -> CachedPointIfma { - CachedPointIfma(FieldElement51x4([ - _mm256_loadu_si256(entry[0].as_ptr().cast()), - _mm256_loadu_si256(entry[1].as_ptr().cast()), - _mm256_loadu_si256(entry[2].as_ptr().cast()), - _mm256_loadu_si256(entry[3].as_ptr().cast()), - _mm256_loadu_si256(entry[4].as_ptr().cast()), - ])) +#[cfg(any(test, feature = "ed25519"))] +fn load_cached_ifma_raw(entry: &[[i64; 4]; 5]) -> CachedPointIfma { + // SAFETY: AVX2 is active in this function, and every inner array provides 32 initialized bytes for an unaligned load. + let limbs = unsafe { + [ + _mm256_loadu_si256(entry[0].as_ptr().cast()), + _mm256_loadu_si256(entry[1].as_ptr().cast()), + _mm256_loadu_si256(entry[2].as_ptr().cast()), + _mm256_loadu_si256(entry[3].as_ptr().cast()), + _mm256_loadu_si256(entry[4].as_ptr().cast()), + ] + }; + CachedPointIfma(FieldElement51x4(limbs)) } /// Add a signed wNAF digit from the static raw basepoint table. @@ -1003,8 +990,8 @@ unsafe fn load_cached_ifma_raw(entry: &[[i64; 4]; 5]) -> CachedPointIfma { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add_wnaf_digit_ifma_raw(acc: ExtendedPointIfma, table: &[[[i64; 4]; 5]], digit: i8) -> ExtendedPointIfma { +#[cfg(any(test, feature = "ed25519"))] +fn add_wnaf_digit_ifma_raw(acc: ExtendedPointIfma, table: &[[[i64; 4]; 5]], digit: i8) -> ExtendedPointIfma { let index = usize::from((digit.unsigned_abs().wrapping_sub(1)) / 2); let Some(entry) = table.get(index) else { return acc; @@ -1033,8 +1020,7 @@ unsafe fn add_wnaf_digit_ifma_raw(acc: ExtendedPointIfma, table: &[[[i64; 4]; 5] /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -#[allow(clippy::indexing_slicing)] // i bounded by top < 256, naf arrays are [i8; 256] +#[cfg(feature = "ed25519")] pub(crate) unsafe fn straus_wnaf_vartime_ifma(s: &[u8; 32], h: &[u8; 32], a: &ExtendedPoint) -> ExtendedPoint { let s_naf = scalar::non_adjacent_form(s, 8); let h_naf = scalar::non_adjacent_form(h, 5); @@ -1096,20 +1082,37 @@ mod tests { ExtendedPoint::basepoint() } - fn decode_hex_32(hex: &str) -> [u8; 32] { - let bytes = hex.as_bytes(); + fn decode_hex_32(hex: &str) -> Option<[u8; 32]> { let mut out = [0u8; 32]; - for (dst, chunk) in out.iter_mut().zip(bytes.chunks_exact(2)) { - *dst = hex_val(chunk[0]) << 4 | hex_val(chunk[1]); - } - out + crate::hex::from_hex(hex, &mut out).ok()?; + Some(out) } - fn hex_val(b: u8) -> u8 { - match b { - b'0'..=b'9' => b - b'0', - b'a'..=b'f' => b - b'a' + 10, - _ => panic!("invalid hex"), + #[test] + fn ifma_wnaf8_table_matches_portable_odd_multiples() { + let step = basepoint().double(); + let mut point = basepoint(); + let d2 = FieldElement::from_small(D2); + let d2_twice = FieldElement::from_small(D2.wrapping_mul(2)); + let d1_twice = FieldElement::from_small(D1.wrapping_mul(2)); + + for entry in &basepoint_table_ifma::BASEPOINT_WNAF8_IFMA_RAW { + let (x, y, z, t) = point.components(); + let a = d2.mul(&y.sub(x)).normalize(); + let b = d2.mul(&y.add(x)).normalize(); + let c = d2_twice.mul(z).normalize(); + let d = d1_twice.mul(t).neg().normalize(); + + for ((((raw, &a_limb), &b_limb), &c_limb), &d_limb) in + entry.iter().zip(a.limbs()).zip(b.limbs()).zip(c.limbs()).zip(d.limbs()) + { + assert_eq!( + (*raw).map(|value| u64::from_ne_bytes(value.to_ne_bytes())), + [a_limb, b_limb, c_limb, d_limb] + ); + } + + point = point.add(&step); } } @@ -1311,9 +1314,9 @@ mod tests { } use crate::auth::ed25519::{Ed25519SecretKey, hash::ExpandedSecret}; - let secret = Ed25519SecretKey::from_bytes(decode_hex_32( - "9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60", - )); + let secret_bytes = decode_hex_32("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60"); + assert!(secret_bytes.is_some()); + let secret = Ed25519SecretKey::from_bytes(secret_bytes.unwrap_or_default()); let expanded = ExpandedSecret::from_secret_key(&secret); let expected = decode_hex_32("d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a"); @@ -1322,7 +1325,7 @@ mod tests { let avx_pub = scalar_mul_basepoint_avx2(expanded.scalar_bytes()); assert_eq!( avx_pub.to_bytes(), - Some(expected), + expected, "AVX2 basepoint mul should match RFC 8032 vector 1" ); } @@ -1379,9 +1382,15 @@ mod tests { r_bytes.copy_from_slice(&sig_bytes[..32]); s_bytes.copy_from_slice(&sig_bytes[32..]); - let r_point = ExtendedPoint::from_bytes(&r_bytes).unwrap(); - let a_point = ExtendedPoint::from_bytes(public.as_bytes()).unwrap(); - let s_scalar = super::super::scalar::from_canonical_bytes(&s_bytes).unwrap(); + let r_point = ExtendedPoint::from_bytes(&r_bytes); + let a_point = ExtendedPoint::from_bytes(public.as_bytes()); + let s_scalar = super::super::scalar::from_canonical_bytes(&s_bytes); + assert!(r_point.is_some()); + assert!(a_point.is_some()); + assert!(s_scalar.is_some()); + let r_point = r_point.unwrap_or_default(); + let a_point = a_point.unwrap_or_default(); + let s_scalar = s_scalar.unwrap_or_default(); let mut hasher = Sha512::new(); hasher.update(&r_bytes); diff --git a/src/auth/ed25519/scalar.rs b/src/auth/ed25519/scalar.rs index 9724c1d6..b0866ec9 100644 --- a/src/auth/ed25519/scalar.rs +++ b/src/auth/ed25519/scalar.rs @@ -1,5 +1,3 @@ -#![allow(clippy::identity_op, clippy::indexing_slicing)] - //! Internal Ed25519 scalar arithmetic mod the group order `L`. //! //! This is the correctness-first baseline for signing and verification. It @@ -53,7 +51,30 @@ struct Scalar52([u64; 5]); #[inline(always)] #[must_use] fn wide_mul(lhs: u64, rhs: u64) -> u128 { - u128::from(lhs) * u128::from(rhs) + u128::from(lhs).strict_mul(u128::from(rhs)) +} + +#[inline(always)] +#[must_use] +fn split_u128(value: u128) -> (u64, u64) { + let [b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15] = value.to_le_bytes(); + ( + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]), + u64::from_le_bytes([b8, b9, b10, b11, b12, b13, b14, b15]), + ) +} + +#[inline(always)] +#[must_use] +fn low_u64(value: u128) -> u64 { + split_u128(value).0 +} + +#[inline(always)] +#[must_use] +fn low_u8(value: u64) -> u8 { + let [low, ..] = value.to_le_bytes(); + low } impl Scalar52 { @@ -148,44 +169,20 @@ impl Scalar52 { Self::add(&hi, &lo) } - #[rustfmt::skip] #[must_use] fn as_bytes(&self) -> [u8; 32] { let limbs = &self.0; - [ - ( limbs[0] >> 0) as u8, - ( limbs[0] >> 8) as u8, - ( limbs[0] >> 16) as u8, - ( limbs[0] >> 24) as u8, - ( limbs[0] >> 32) as u8, - ( limbs[0] >> 40) as u8, - ((limbs[0] >> 48) | (limbs[1] << 4)) as u8, - ( limbs[1] >> 4) as u8, - ( limbs[1] >> 12) as u8, - ( limbs[1] >> 20) as u8, - ( limbs[1] >> 28) as u8, - ( limbs[1] >> 36) as u8, - ( limbs[1] >> 44) as u8, - ( limbs[2] >> 0) as u8, - ( limbs[2] >> 8) as u8, - ( limbs[2] >> 16) as u8, - ( limbs[2] >> 24) as u8, - ( limbs[2] >> 32) as u8, - ( limbs[2] >> 40) as u8, - ((limbs[2] >> 48) | (limbs[3] << 4)) as u8, - ( limbs[3] >> 4) as u8, - ( limbs[3] >> 12) as u8, - ( limbs[3] >> 20) as u8, - ( limbs[3] >> 28) as u8, - ( limbs[3] >> 36) as u8, - ( limbs[3] >> 44) as u8, - ( limbs[4] >> 0) as u8, - ( limbs[4] >> 8) as u8, - ( limbs[4] >> 16) as u8, - ( limbs[4] >> 24) as u8, - ( limbs[4] >> 32) as u8, - ( limbs[4] >> 40) as u8, - ] + let words = [ + limbs[0] | (limbs[1] << 52), + (limbs[1] >> 12) | (limbs[2] << 40), + (limbs[2] >> 24) | (limbs[3] << 28), + (limbs[3] >> 36) | (limbs[4] << 16), + ]; + let mut bytes = [0u8; 32]; + for (chunk, word) in bytes.as_chunks_mut::<8>().0.iter_mut().zip(words) { + chunk.copy_from_slice(&word.to_le_bytes()); + } + bytes } #[must_use] @@ -207,7 +204,7 @@ impl Scalar52 { fn barrier(value: u64) -> u64 { // SAFETY: `value` is a local `u64`; reading it through a volatile pointer // preserves the arithmetic shape without violating aliasing or lifetime rules. - unsafe { core::ptr::read_volatile(&value) } + unsafe { core::ptr::read_volatile(core::ptr::from_ref(&value)) } } let mut out = [0u64; 5]; @@ -236,7 +233,7 @@ impl Scalar52 { fn barrier(value: u64) -> u64 { // SAFETY: `value` is a local `u64`; reading it through a volatile pointer // preserves the arithmetic shape without violating aliasing or lifetime rules. - unsafe { core::ptr::read_volatile(&value) } + unsafe { core::ptr::read_volatile(core::ptr::from_ref(&value)) } } let mut out = [0u64; 5]; @@ -261,18 +258,31 @@ impl Scalar52 { result } - #[rustfmt::skip] #[must_use] fn mul_internal(lhs: &Self, rhs: &Self) -> [u128; 9] { [ wide_mul(lhs.0[0], rhs.0[0]), - wide_mul(lhs.0[0], rhs.0[1]) + wide_mul(lhs.0[1], rhs.0[0]), - wide_mul(lhs.0[0], rhs.0[2]) + wide_mul(lhs.0[1], rhs.0[1]) + wide_mul(lhs.0[2], rhs.0[0]), - wide_mul(lhs.0[0], rhs.0[3]) + wide_mul(lhs.0[1], rhs.0[2]) + wide_mul(lhs.0[2], rhs.0[1]) + wide_mul(lhs.0[3], rhs.0[0]), - wide_mul(lhs.0[0], rhs.0[4]) + wide_mul(lhs.0[1], rhs.0[3]) + wide_mul(lhs.0[2], rhs.0[2]) + wide_mul(lhs.0[3], rhs.0[1]) + wide_mul(lhs.0[4], rhs.0[0]), - wide_mul(lhs.0[1], rhs.0[4]) + wide_mul(lhs.0[2], rhs.0[3]) + wide_mul(lhs.0[3], rhs.0[2]) + wide_mul(lhs.0[4], rhs.0[1]), - wide_mul(lhs.0[2], rhs.0[4]) + wide_mul(lhs.0[3], rhs.0[3]) + wide_mul(lhs.0[4], rhs.0[2]), - wide_mul(lhs.0[3], rhs.0[4]) + wide_mul(lhs.0[4], rhs.0[3]), + wide_mul(lhs.0[0], rhs.0[1]).strict_add(wide_mul(lhs.0[1], rhs.0[0])), + wide_mul(lhs.0[0], rhs.0[2]) + .strict_add(wide_mul(lhs.0[1], rhs.0[1])) + .strict_add(wide_mul(lhs.0[2], rhs.0[0])), + wide_mul(lhs.0[0], rhs.0[3]) + .strict_add(wide_mul(lhs.0[1], rhs.0[2])) + .strict_add(wide_mul(lhs.0[2], rhs.0[1])) + .strict_add(wide_mul(lhs.0[3], rhs.0[0])), + wide_mul(lhs.0[0], rhs.0[4]) + .strict_add(wide_mul(lhs.0[1], rhs.0[3])) + .strict_add(wide_mul(lhs.0[2], rhs.0[2])) + .strict_add(wide_mul(lhs.0[3], rhs.0[1])) + .strict_add(wide_mul(lhs.0[4], rhs.0[0])), + wide_mul(lhs.0[1], rhs.0[4]) + .strict_add(wide_mul(lhs.0[2], rhs.0[3])) + .strict_add(wide_mul(lhs.0[3], rhs.0[2])) + .strict_add(wide_mul(lhs.0[4], rhs.0[1])), + wide_mul(lhs.0[2], rhs.0[4]) + .strict_add(wide_mul(lhs.0[3], rhs.0[3])) + .strict_add(wide_mul(lhs.0[4], rhs.0[2])), + wide_mul(lhs.0[3], rhs.0[4]).strict_add(wide_mul(lhs.0[4], rhs.0[3])), wide_mul(lhs.0[4], rhs.0[4]), ] } @@ -298,13 +308,13 @@ impl Scalar52 { fn montgomery_reduce_unreduced(limbs: &[u128; 9]) -> Self { #[inline(always)] fn part1(sum: u128) -> (u128, u64) { - let p = (sum as u64).wrapping_mul(LFACTOR52) & RADIX52_MASK; + let p = low_u64(sum).wrapping_mul(LFACTOR52) & RADIX52_MASK; ((sum.strict_add(wide_mul(p, ORDER52.0[0]))) >> 52, p) } #[inline(always)] fn part2(sum: u128) -> (u128, u64) { - let word = (sum as u64) & RADIX52_MASK; + let word = low_u64(sum) & RADIX52_MASK; (sum >> 52, word) } @@ -345,7 +355,7 @@ impl Scalar52 { ); let (carry, r2) = part2(carry.strict_add(limbs[7]).strict_add(wide_mul(n3, ORDER52.0[4]))); let (carry, r3) = part2(carry.strict_add(limbs[8]).strict_add(wide_mul(n4, ORDER52.0[4]))); - let r4 = carry as u64; + let r4 = low_u64(carry); Self([r0, r1, r2, r3, r4]) } @@ -413,7 +423,7 @@ pub(crate) fn clamp_secret_scalar(bytes: &mut [u8; SECRET_KEY_LENGTH]) { #[must_use] pub(crate) fn decode_words_le(bytes: &[u8; SECRET_KEY_LENGTH]) -> Scalar { let mut limbs = [0u64; SCALAR_LIMBS]; - for (limb, chunk) in limbs.iter_mut().zip(bytes.as_slice().chunks_exact(8)) { + for (limb, chunk) in limbs.iter_mut().zip(bytes.as_chunks::<8>().0) { *limb = read_u64_le(chunk); } limbs @@ -434,15 +444,8 @@ pub(crate) fn from_canonical_bytes(bytes: &[u8; SECRET_KEY_LENGTH]) -> Option [u8; SECRET_KEY_LENGTH] { let mut out = [0u8; SECRET_KEY_LENGTH]; - for (chunk, limb) in out.as_mut_slice().chunks_exact_mut(8).zip(words.iter().copied()) { - chunk[0] = limb as u8; - chunk[1] = (limb >> 8) as u8; - chunk[2] = (limb >> 16) as u8; - chunk[3] = (limb >> 24) as u8; - chunk[4] = (limb >> 32) as u8; - chunk[5] = (limb >> 40) as u8; - chunk[6] = (limb >> 48) as u8; - chunk[7] = (limb >> 56) as u8; + for (chunk, limb) in out.as_chunks_mut::<8>().0.iter_mut().zip(words.iter().copied()) { + chunk.copy_from_slice(&limb.to_le_bytes()); } out } @@ -521,28 +524,6 @@ pub(crate) fn negate_mod(s: &Scalar) -> Scalar { } } -/// Decompose a scalar encoding into signed radix-16 digits in `[-8, 8]`. -#[must_use] -#[allow(clippy::indexing_slicing)] -pub(crate) fn as_radix_16(bytes: &[u8; SECRET_KEY_LENGTH]) -> [i8; 64] { - debug_assert!(bytes[31] <= 127); - - let mut digits = [0i8; 64]; - - for (i, byte) in bytes.iter().copied().enumerate() { - digits[2 * i] = (byte & 0x0F) as i8; - digits[2 * i + 1] = ((byte >> 4) & 0x0F) as i8; - } - - for i in 0..63 { - let carry = (digits[i] + 8) >> 4; - digits[i] -= carry << 4; - digits[i + 1] += carry; - } - - digits -} - /// Decompose a scalar into width-`w` non-adjacent form (wNAF). /// /// Returns a 256-element array of signed digits. Non-zero digits are odd @@ -556,7 +537,6 @@ pub(crate) fn as_radix_16(bytes: &[u8; SECRET_KEY_LENGTH]) -> [i8; 64] { /// /// Debug-panics if `w < 2` or `w > 8`. #[must_use] -#[allow(clippy::indexing_slicing)] pub(crate) fn non_adjacent_form(bytes: &[u8; 32], w: usize) -> [i8; 256] { debug_assert!((2..=8).contains(&w)); @@ -565,7 +545,7 @@ pub(crate) fn non_adjacent_form(bytes: &[u8; 32], w: usize) -> [i8; 256] { // Load scalar into mutable u64 words. The 5th word absorbs carry from // the 256th bit position. let mut x = [0u64; 5]; - for (dst, chunk) in x.iter_mut().zip(bytes.as_slice().chunks_exact(8)) { + for (dst, chunk) in x.iter_mut().zip(bytes.as_chunks::<8>().0) { *dst = read_u64_le(chunk); } @@ -597,10 +577,10 @@ pub(crate) fn non_adjacent_form(bytes: &[u8; 32], w: usize) -> [i8; 256] { // Odd window: emit a signed digit. if window < width / 2 { - naf[pos] = window as i8; + naf[pos] = i8::from_ne_bytes([low_u8(window)]); carry = 0; } else { - naf[pos] = (window as i8).wrapping_sub(width as i8); + naf[pos] = i8::from_ne_bytes([low_u8(window).wrapping_sub(low_u8(width))]); carry = 1; } @@ -613,22 +593,15 @@ pub(crate) fn non_adjacent_form(bytes: &[u8; 32], w: usize) -> [i8; 256] { #[inline] #[must_use] -fn read_u64_le(chunk: &[u8]) -> u64 { - u64::from(chunk[0]) - | (u64::from(chunk[1]) << 8) - | (u64::from(chunk[2]) << 16) - | (u64::from(chunk[3]) << 24) - | (u64::from(chunk[4]) << 32) - | (u64::from(chunk[5]) << 40) - | (u64::from(chunk[6]) << 48) - | (u64::from(chunk[7]) << 56) +fn read_u64_le(chunk: &[u8; 8]) -> u64 { + u64::from_le_bytes(*chunk) } #[inline] #[must_use] fn read_words_le_32(bytes: &[u8; 32]) -> [u64; 4] { let mut out = [0u64; 4]; - for (dst, chunk) in out.iter_mut().zip(bytes.as_slice().chunks_exact(8)) { + for (dst, chunk) in out.iter_mut().zip(bytes.as_chunks::<8>().0) { *dst = read_u64_le(chunk); } out @@ -638,7 +611,7 @@ fn read_words_le_32(bytes: &[u8; 32]) -> [u64; 4] { #[must_use] fn read_words_le_64(bytes: &[u8; 64]) -> [u64; 8] { let mut out = [0u64; 8]; - for (dst, chunk) in out.iter_mut().zip(bytes.as_slice().chunks_exact(8)) { + for (dst, chunk) in out.iter_mut().zip(bytes.as_chunks::<8>().0) { *dst = read_u64_le(chunk); } out @@ -710,15 +683,16 @@ fn compare(lhs: &Scalar, rhs: &Scalar) -> Ordering { #[must_use] fn add_raw(lhs: &Scalar, rhs: &Scalar) -> (Scalar, u64) { let mut out = ZERO; - let mut carry = 0u128; + let mut carry = 0u64; for (dst, (&left, &right)) in out.iter_mut().zip(lhs.iter().zip(rhs.iter())) { - let sum = u128::from(left).strict_add(u128::from(right)).strict_add(carry); - *dst = sum as u64; - carry = sum >> 64; + let sum = u128::from(left) + .strict_add(u128::from(right)) + .strict_add(u128::from(carry)); + (*dst, carry) = split_u128(sum); } - (out, carry as u64) + (out, carry) } #[inline] @@ -751,8 +725,8 @@ fn maybe_sub_order(words: Scalar) -> Scalar { #[cfg(test)] mod tests { use super::{ - ORDER, Scalar, add_mod, as_radix_16, clamp_secret_scalar, decode_words_le, from_canonical_bytes, mul_add_mod, - mul_add_mod_secret, non_adjacent_form, reduce_64_bytes_mod_order_secret, reduce_bytes_mod_order, + ORDER, Scalar, add_mod, clamp_secret_scalar, decode_words_le, from_canonical_bytes, mul_add_mod, + mul_add_mod_secret, non_adjacent_form, read_u64_le, reduce_64_bytes_mod_order_secret, reduce_bytes_mod_order, reduce_bytes_mod_order_fallback, to_bytes, }; @@ -760,6 +734,14 @@ mod tests { [value, 0, 0, 0] } + fn patterned_bytes(mut value: u8, step: u8) -> [u8; 64] { + core::array::from_fn(|_| { + let current = value; + value = value.wrapping_add(step); + current + }) + } + #[test] fn clamp_secret_scalar_matches_rfc_8032_bit_rules() { let mut bytes = [0xFFu8; 32]; @@ -816,14 +798,14 @@ mod tests { #[test] fn wide_reduction_matches_fallback_for_fixed_input() { - let bytes = core::array::from_fn::<_, 64, _>(|i| (i as u8).wrapping_mul(17).wrapping_add(9)); + let bytes = patterned_bytes(9, 17); assert_eq!(reduce_bytes_mod_order(&bytes), reduce_bytes_mod_order_fallback(&bytes)); } #[test] fn secret_wide_reduction_matches_public_reduction() { - let bytes = core::array::from_fn::<_, 64, _>(|i| (i as u8).wrapping_mul(29).wrapping_add(13)); + let bytes = patterned_bytes(13, 29); assert_eq!(reduce_64_bytes_mod_order_secret(&bytes), reduce_bytes_mod_order(&bytes)); } @@ -832,7 +814,7 @@ mod tests { fn radix16_recenters_nibbles_into_signed_digits() { let mut bytes = [0u8; 32]; bytes[0] = 0x19; - let digits = as_radix_16(&bytes); + let digits = super::super::scalar_radix_16(&bytes); assert_eq!(digits[0], -7); assert_eq!(digits[1], 2); @@ -843,7 +825,7 @@ mod tests { #[test] fn modular_addition_wraps_order_boundary() { - let near_order = [ORDER[0] - 1, ORDER[1], ORDER[2], ORDER[3]]; + let near_order = [ORDER[0].strict_sub(1), ORDER[1], ORDER[2], ORDER[3]]; let wrapped = add_mod(&near_order, &from_u64(1)); assert_eq!(wrapped, [0, 0, 0, 0]); @@ -907,7 +889,7 @@ mod tests { let word = i / 64; let bit = i % 64; if digit > 0 { - let d = digit as u64; + let d = u64::from(digit.unsigned_abs()); let (new_val, carry) = reconstructed[word].overflowing_add(d << bit); reconstructed[word] = new_val; if carry { @@ -920,11 +902,12 @@ mod tests { } } // Handle cross-word shift - if bit > 0 && word + 1 < 5 { - reconstructed[word + 1] = reconstructed[word + 1].wrapping_add(d >> (64 - bit)); + let next_word = word.strict_add(1); + if bit > 0 && next_word < 5 { + reconstructed[next_word] = reconstructed[next_word].wrapping_add(d >> 64usize.strict_sub(bit)); } } else { - let d = (-digit) as u64; + let d = u64::from(digit.unsigned_abs()); let (new_val, borrow) = reconstructed[word].overflowing_sub(d << bit); reconstructed[word] = new_val; if borrow { @@ -936,16 +919,17 @@ mod tests { } } } - if bit > 0 && word + 1 < 5 { - reconstructed[word + 1] = reconstructed[word + 1].wrapping_sub(d >> (64 - bit)); + let next_word = word.strict_add(1); + if bit > 0 && next_word < 5 { + reconstructed[next_word] = reconstructed[next_word].wrapping_sub(d >> 64usize.strict_sub(bit)); } } } // Compare lower 4 words (256 bits) let mut expected = [0u64; 4]; - for (dst, chunk) in expected.iter_mut().zip(scalar_bytes.chunks_exact(8)) { - *dst = u64::from_le_bytes(chunk.try_into().unwrap()); + for (dst, chunk) in expected.iter_mut().zip(scalar_bytes.as_chunks::<8>().0) { + *dst = read_u64_le(chunk); } assert_eq!(&reconstructed[..4], &expected, "wNAF({w}) reconstruction failed"); } @@ -962,7 +946,7 @@ mod tests { for w in [5, 8] { let naf = non_adjacent_form(&s, w); - let max_abs = (1i16 << (w - 1)) - 1; // 2^(w-1) - 1 + let max_abs = (1i16 << w.strict_sub(1)).strict_sub(1); // 2^(w-1) - 1 for (i, &digit) in naf.iter().enumerate() { if digit != 0 { assert!(digit.abs() % 2 == 1, "wNAF({w}) digit[{i}] = {digit} is even"); diff --git a/src/auth/ed25519/x86_64_asm.rs b/src/auth/ed25519/x86_64_asm.rs index ed77a964..c0ac29d1 100644 --- a/src/auth/ed25519/x86_64_asm.rs +++ b/src/auth/ed25519/x86_64_asm.rs @@ -4,8 +4,6 @@ //! fixed-base multiplication backend. This module owns the ABI boundary; //! `ed25519.rs` owns signing semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; use super::{ diff --git a/src/auth/hkdf.rs b/src/auth/hkdf.rs index 7c2c29be..f8a6864e 100644 --- a/src/auth/hkdf.rs +++ b/src/auth/hkdf.rs @@ -25,7 +25,6 @@ const SHA512_MAX_OUTPUT_SIZE: usize = 255 * SHA512_OUTPUT_SIZE; const SHA512_BLOCK_SIZE: usize = 128; #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_u32x8_be(dst: &mut [u8; SHA256_OUTPUT_SIZE], words: &[u32; 8]) { dst[0..4].copy_from_slice(&words[0].to_be_bytes()); dst[4..8].copy_from_slice(&words[1].to_be_bytes()); @@ -38,7 +37,6 @@ fn write_u32x8_be(dst: &mut [u8; SHA256_OUTPUT_SIZE], words: &[u32; 8]) { } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_u64x6_be(dst: &mut [u8], words: &[u64; 8]) { debug_assert!(dst.len() >= SHA384_OUTPUT_SIZE); dst[0..8].copy_from_slice(&words[0].to_be_bytes()); @@ -50,7 +48,6 @@ fn write_u64x6_be(dst: &mut [u8], words: &[u64; 8]) { } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_u64x8_be(dst: &mut [u8], words: &[u64; 8]) { debug_assert!(dst.len() >= SHA512_OUTPUT_SIZE); dst[0..8].copy_from_slice(&words[0].to_be_bytes()); @@ -173,7 +170,6 @@ impl HkdfSha256 { /// bypassing all `Sha256` struct creation, `Drop` zeroization, and dispatch /// overhead in the inner loop. #[inline] - #[allow(clippy::indexing_slicing)] pub fn expand(&self, info: &[u8], okm: &mut [u8]) -> Result<(), HkdfOutputLengthError> { if okm.len() > SHA256_MAX_OUTPUT_SIZE { return Err(HkdfOutputLengthError::new()); @@ -326,6 +322,7 @@ impl HkdfSha256 { #[cfg(feature = "diag")] #[unsafe(no_mangle)] #[inline(never)] +/// Derive a portable HKDF-SHA256 diagnostic output using fixed `b"salt"` and `b"info"` inputs. pub fn diag_hkdf_sha256_derive_portable(input_key_material: &[u8; SHA256_OUTPUT_SIZE]) -> [u8; SHA256_OUTPUT_SIZE] { let compress = crate::hashes::crypto::sha256::kernels::compress_blocks_fn( crate::hashes::crypto::sha256::kernels::Sha256KernelId::Portable, @@ -428,7 +425,6 @@ impl HkdfSha384 { /// bypassing all `Sha384` struct creation, `Drop` zeroization, and dispatch /// overhead in the inner loop. #[inline] - #[allow(clippy::indexing_slicing)] pub fn expand(&self, info: &[u8], okm: &mut [u8]) -> Result<(), HkdfOutputLengthError> { if okm.len() > SHA384_MAX_OUTPUT_SIZE { return Err(HkdfOutputLengthError::new()); @@ -565,6 +561,7 @@ impl HkdfSha384 { #[cfg(feature = "diag")] #[unsafe(no_mangle)] #[inline(never)] +/// Derive a portable HKDF-SHA384 diagnostic output using fixed `b"salt"` and `b"info"` inputs. pub fn diag_hkdf_sha384_derive_portable(input_key_material: &[u8; SHA384_OUTPUT_SIZE]) -> [u8; SHA384_OUTPUT_SIZE] { let compress = crate::hashes::crypto::sha384::kernels::compress_blocks_fn( crate::hashes::crypto::sha384::kernels::Sha384KernelId::Portable, @@ -648,7 +645,6 @@ impl HkdfSha512 { /// Expand the stored pseudorandom key into `okm`. #[inline] - #[allow(clippy::indexing_slicing)] pub fn expand(&self, info: &[u8], okm: &mut [u8]) -> Result<(), HkdfOutputLengthError> { if okm.len() > SHA512_MAX_OUTPUT_SIZE { return Err(HkdfOutputLengthError::new()); @@ -785,6 +781,7 @@ impl HkdfSha512 { #[cfg(feature = "diag")] #[unsafe(no_mangle)] #[inline(never)] +/// Derive a portable HKDF-SHA512 diagnostic output using fixed `b"salt"` and `b"info"` inputs. pub fn diag_hkdf_sha512_derive_portable(input_key_material: &[u8; SHA512_OUTPUT_SIZE]) -> [u8; SHA512_OUTPUT_SIZE] { let compress = crate::hashes::crypto::sha512::kernels::compress_blocks_fn( crate::hashes::crypto::sha512::kernels::Sha512KernelId::Portable, @@ -806,7 +803,6 @@ impl Drop for HkdfSha512 { } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn expand_hmac_sha256_inner( compress: Sha256CompressBlocksFn, prev: Option<&[u8; SHA256_OUTPUT_SIZE]>, @@ -863,7 +859,6 @@ fn expand_hmac_sha256_inner( } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn expand_hmac_sha256_outer( compress: Sha256CompressBlocksFn, outer_init: &[u32; 8], @@ -880,7 +875,6 @@ fn expand_hmac_sha256_outer( } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn expand_hmac_sha384_inner( compress: Sha384CompressBlocksFn, prev: Option<&[u8; SHA384_OUTPUT_SIZE]>, @@ -937,7 +931,6 @@ fn expand_hmac_sha384_inner( } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn expand_hmac_sha384_outer( compress: Sha384CompressBlocksFn, outer_init: &[u64; 8], @@ -952,7 +945,6 @@ fn expand_hmac_sha384_outer( } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn expand_hmac_sha512_inner( compress: Sha512CompressBlocksFn, prev: Option<&[u8; SHA512_OUTPUT_SIZE]>, @@ -1009,7 +1001,6 @@ fn expand_hmac_sha512_inner( } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn expand_hmac_sha512_outer( compress: Sha512CompressBlocksFn, outer_init: &[u64; 8], @@ -1052,9 +1043,9 @@ mod tests { fn pattern(len: usize, mul: u8, add: u8) -> Vec { (0..len) .map(|i| { - (i as u8) + i.to_le_bytes()[0] .wrapping_mul(mul) - .wrapping_add(((i >> 2) as u8).wrapping_add(add)) + .wrapping_add((i >> 2).to_le_bytes()[0].wrapping_add(add)) }) .collect() } @@ -1079,13 +1070,15 @@ mod tests { let mut expected = vec![0u8; out_len]; RustCryptoHkdfSha256::new(Some(&salt), &ikm) .expand(&info, &mut expected) - .unwrap(); + .expect("test output length should be within the HKDF-SHA256 limit"); let public = HkdfSha256::new(&salt, &ikm); let forced = HkdfSha256::extract_with_compress_for_test(&salt, &ikm, compress); let mut public_out = vec![0u8; out_len]; - public.expand(&info, &mut public_out).unwrap(); + public + .expand(&info, &mut public_out) + .expect("test output length should be within the HKDF-SHA256 limit"); assert_eq!( public_out, expected, @@ -1098,7 +1091,9 @@ mod tests { ); let mut forced_out = vec![0u8; out_len]; - forced.expand(&info, &mut forced_out).unwrap(); + forced + .expand(&info, &mut forced_out) + .expect("test output length should be within the HKDF-SHA256 limit"); assert_eq!( forced_out, expected, @@ -1137,13 +1132,15 @@ mod tests { let mut expected = vec![0u8; out_len]; RustCryptoHkdfSha384::new(Some(&salt), &ikm) .expand(&info, &mut expected) - .unwrap(); + .expect("test output length should be within the HKDF-SHA384 limit"); let public = HkdfSha384::new(&salt, &ikm); let forced = HkdfSha384::extract_with_compress_for_test(&salt, &ikm, compress); let mut public_out = vec![0u8; out_len]; - public.expand(&info, &mut public_out).unwrap(); + public + .expand(&info, &mut public_out) + .expect("test output length should be within the HKDF-SHA384 limit"); assert_eq!( public_out, expected, @@ -1156,7 +1153,9 @@ mod tests { ); let mut forced_out = vec![0u8; out_len]; - forced.expand(&info, &mut forced_out).unwrap(); + forced + .expand(&info, &mut forced_out) + .expect("test output length should be within the HKDF-SHA384 limit"); assert_eq!( forced_out, expected, @@ -1195,13 +1194,15 @@ mod tests { let mut expected = vec![0u8; out_len]; RustCryptoHkdfSha512::new(Some(&salt), &ikm) .expand(&info, &mut expected) - .unwrap(); + .expect("test output length should be within the HKDF-SHA512 limit"); let public = HkdfSha512::new(&salt, &ikm); let forced = HkdfSha512::extract_with_compress_for_test(&salt, &ikm, compress); let mut public_out = vec![0u8; out_len]; - public.expand(&info, &mut public_out).unwrap(); + public + .expand(&info, &mut public_out) + .expect("test output length should be within the HKDF-SHA512 limit"); assert_eq!( public_out, expected, @@ -1214,7 +1215,9 @@ mod tests { ); let mut forced_out = vec![0u8; out_len]; - forced.expand(&info, &mut forced_out).unwrap(); + forced + .expand(&info, &mut forced_out) + .expect("test output length should be within the HKDF-SHA512 limit"); assert_eq!( forced_out, expected, diff --git a/src/auth/hmac.rs b/src/auth/hmac.rs index 2458d623..ef4a2b58 100644 --- a/src/auth/hmac.rs +++ b/src/auth/hmac.rs @@ -392,8 +392,7 @@ impl HmacSha256 { } } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(any(feature = "diag", all(test, feature = "hkdf")))] pub(crate) fn new_with_compress_for_test( key: &[u8], compress: crate::hashes::crypto::sha256::kernels::CompressBlocksFn, @@ -427,8 +426,7 @@ impl HmacSha256 { } } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(any(feature = "diag", all(test, feature = "hkdf")))] pub(crate) fn mac_with_compress_for_test( key: &[u8], data: &[u8], @@ -441,6 +439,7 @@ impl HmacSha256 { } #[cfg(feature = "diag")] +/// Compare the portable HMAC-SHA256 tag for the diagnostic message `b"binsec"` with `expected`. pub fn diag_hmac_sha256_verify_portable( key: &[u8; SHA256_TAG_SIZE], expected: &[u8; SHA256_TAG_SIZE], @@ -453,6 +452,7 @@ pub fn diag_hmac_sha256_verify_portable( } #[cfg(feature = "diag")] +/// Compare the leading 64 bits of the portable HMAC-SHA256 diagnostic tag with `expected`. pub fn diag_hmac_sha256_verify_truncated_64_portable( key: &[u8; SHA256_TAG_SIZE], expected: &[u8; 8], @@ -545,7 +545,6 @@ impl Mac for HmacSha256 { /// dispatch, state save/restore) that dominates on fast SHA2-CE cores. /// The outer hash is always merged into a single 128-byte (2-block) call. #[inline] - #[allow(clippy::indexing_slicing)] // All indices bounded by prior length checks + fixed-size arrays. fn mac(key: &[u8], data: &[u8]) -> Self::Tag { if hmac_sha256_oneshot_should_stream() { // The Sapphire Rapids policy uses the public streaming path instead of @@ -598,7 +597,12 @@ impl Mac for HmacSha256 { 256 => compress_inline_inner!(256), 320 => compress_inline_inner!(320), 384 => compress_inline_inner!(384), - _ => unreachable!("HMAC-SHA256 inline inner padding is bounded to 128..=384 bytes"), + _ => { + ct::zeroize(&mut ipad); + let mut mac = Self::new(key); + mac.update(data); + return mac.finalize(); + } } } else { compress(&mut state, &ipad); @@ -634,7 +638,7 @@ impl Mac for HmacSha256 { compress(&mut state, &outer); let mut tag = [0u8; SHA256_TAG_SIZE]; - for (chunk, &word) in tag.chunks_exact_mut(4).zip(state.iter()) { + for (chunk, &word) in tag.as_chunks_mut::<4>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } @@ -768,6 +772,7 @@ impl HmacSha384 { } #[cfg(feature = "diag")] +/// Compare the portable HMAC-SHA384 tag for the diagnostic message `b"binsec"` with `expected`. pub fn diag_hmac_sha384_verify_portable( key: &[u8; SHA384_TAG_SIZE], expected: &[u8; SHA384_TAG_SIZE], @@ -841,7 +846,6 @@ impl Mac for HmacSha384 { } #[inline] - #[allow(clippy::indexing_slicing)] // All indices bounded by prior length checks + fixed-size arrays. fn mac(key: &[u8], data: &[u8]) -> Self::Tag { const INLINE_DATA_MAX: usize = 256; @@ -884,7 +888,12 @@ impl Mac for HmacSha384 { 256 => compress_inline_inner!(256), 384 => compress_inline_inner!(384), 512 => compress_inline_inner!(512), - _ => unreachable!("HMAC-SHA384 inline inner padding is bounded to 256..=512 bytes"), + _ => { + ct::zeroize(&mut ipad); + let mut mac = Self::new(key); + mac.update(data); + return mac.finalize(); + } } } else { compress(&mut state, &ipad); @@ -920,7 +929,7 @@ impl Mac for HmacSha384 { compress(&mut state, &outer); let mut tag = [0u8; SHA384_TAG_SIZE]; - for (chunk, &word) in tag.chunks_exact_mut(8).zip(state.iter()) { + for (chunk, &word) in tag.as_chunks_mut::<8>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } @@ -1054,6 +1063,7 @@ impl HmacSha512 { } #[cfg(feature = "diag")] +/// Compare the portable HMAC-SHA512 tag for the diagnostic message `b"binsec"` with `expected`. pub fn diag_hmac_sha512_verify_portable( key: &[u8; SHA512_TAG_SIZE], expected: &[u8; SHA512_TAG_SIZE], @@ -1127,7 +1137,6 @@ impl Mac for HmacSha512 { } #[inline] - #[allow(clippy::indexing_slicing)] // All indices bounded by prior length checks + fixed-size arrays. fn mac(key: &[u8], data: &[u8]) -> Self::Tag { const INLINE_DATA_MAX: usize = 256; @@ -1170,7 +1179,12 @@ impl Mac for HmacSha512 { 256 => compress_inline_inner!(256), 384 => compress_inline_inner!(384), 512 => compress_inline_inner!(512), - _ => unreachable!("HMAC-SHA512 inline inner padding is bounded to 256..=512 bytes"), + _ => { + ct::zeroize(&mut ipad); + let mut mac = Self::new(key); + mac.update(data); + return mac.finalize(); + } } } else { compress(&mut state, &ipad); @@ -1206,7 +1220,7 @@ impl Mac for HmacSha512 { compress(&mut state, &outer); let mut tag = [0u8; SHA512_TAG_SIZE]; - for (chunk, &word) in tag.chunks_exact_mut(8).zip(state.iter()) { + for (chunk, &word) in tag.as_chunks_mut::<8>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } @@ -1263,15 +1277,15 @@ mod tests { fn pattern(len: usize, mul: u8, add: u8) -> Vec { (0..len) .map(|i| { - (i as u8) + i.to_le_bytes()[0] .wrapping_mul(mul) - .wrapping_add(((i >> 3) as u8).wrapping_add(add)) + .wrapping_add((i >> 3).to_le_bytes()[0].wrapping_add(add)) }) .collect() } fn oracle_hmac_sha384(key: &[u8], data: &[u8]) -> [u8; SHA384_TAG_SIZE] { - let mut mac = RustCryptoHmacSha384::new_from_slice(key).unwrap(); + let mut mac = RustCryptoHmacSha384::new_from_slice(key).expect("HMAC accepts keys of any length"); mac.update(data); let bytes = mac.finalize().into_bytes(); let mut tag = [0u8; SHA384_TAG_SIZE]; @@ -1280,7 +1294,7 @@ mod tests { } fn oracle_hmac_sha512(key: &[u8], data: &[u8]) -> [u8; SHA512_TAG_SIZE] { - let mut mac = RustCryptoHmacSha512::new_from_slice(key).unwrap(); + let mut mac = RustCryptoHmacSha512::new_from_slice(key).expect("HMAC accepts keys of any length"); mac.update(data); let bytes = mac.finalize().into_bytes(); let mut tag = [0u8; SHA512_TAG_SIZE]; diff --git a/src/auth/hmac_sha3.rs b/src/auth/hmac_sha3.rs index a88bd445..d2d1637d 100644 --- a/src/auth/hmac_sha3.rs +++ b/src/auth/hmac_sha3.rs @@ -1,7 +1,5 @@ //! HMAC-SHA3 family (RFC 2104, FIPS 198-1, FIPS 202). -#![allow(clippy::indexing_slicing)] // HMAC pads are fixed-size arrays bounded by rate constants. - use crate::{ hashes::crypto::keccak::KeccakCore, traits::{Mac, VerificationError, ct}, diff --git a/src/auth/kmac.rs b/src/auth/kmac.rs index c2ea51ad..b6639675 100644 --- a/src/auth/kmac.rs +++ b/src/auth/kmac.rs @@ -1,7 +1,5 @@ //! KMAC128 and KMAC256 (SP 800-185). -#![allow(clippy::indexing_slicing)] // Fixed-size scratch buffers and encoded suffix slices. - use core::fmt; use crate::{ @@ -190,6 +188,7 @@ define_kmac!(Kmac256, Cshake256, "256", 32); #[cfg(test)] mod tests { use super::{Kmac128, Kmac256}; + use crate::traits::VerificationError; #[test] fn reset_restores_keyed_state() { @@ -226,11 +225,16 @@ mod tests { ] { let mut tag = [0u8; Kmac128::MIN_AUTH_TAG_SIZE + 1]; Kmac128::mac_into(KEY, CUSTOMIZATION, MESSAGE, &mut tag[..len]); + let expected = if len >= Kmac128::MIN_AUTH_TAG_SIZE { + Ok(()) + } else { + Err(VerificationError::new()) + }; + assert_eq!(Kmac128::verify_tag(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]), expected); assert_eq!( - Kmac128::verify_tag(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]).is_ok(), - len >= Kmac128::MIN_AUTH_TAG_SIZE + Kmac128::verify_tag_primitive(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]), + Ok(()) ); - assert!(Kmac128::verify_tag_primitive(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]).is_ok()); } for len in [ @@ -240,11 +244,16 @@ mod tests { ] { let mut tag = [0u8; Kmac256::MIN_AUTH_TAG_SIZE + 1]; Kmac256::mac_into(KEY, CUSTOMIZATION, MESSAGE, &mut tag[..len]); + let expected = if len >= Kmac256::MIN_AUTH_TAG_SIZE { + Ok(()) + } else { + Err(VerificationError::new()) + }; + assert_eq!(Kmac256::verify_tag(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]), expected); assert_eq!( - Kmac256::verify_tag(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]).is_ok(), - len >= Kmac256::MIN_AUTH_TAG_SIZE + Kmac256::verify_tag_primitive(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]), + Ok(()) ); - assert!(Kmac256::verify_tag_primitive(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]).is_ok()); } } } diff --git a/src/auth/mlkem.rs b/src/auth/mlkem.rs index 1f3ebc44..763bbecf 100644 --- a/src/auth/mlkem.rs +++ b/src/auth/mlkem.rs @@ -815,7 +815,7 @@ macro_rules! impl_mlkem_profile_ops { >( decapsulation_key.as_bytes(), ciphertext.as_bytes(), - ))) + )?)) } } }; diff --git a/src/auth/mlkem/portable.rs b/src/auth/mlkem/portable.rs index 50865676..76636042 100644 --- a/src/auth/mlkem/portable.rs +++ b/src/auth/mlkem/portable.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] // Fixed-size FIPS buffers and public loop indices bound every access. - #[cfg(all( target_arch = "aarch64", any(target_os = "macos", target_os = "linux"), @@ -12,13 +10,6 @@ mod s390x; #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] mod x86_64; -#[cfg(all( - target_arch = "aarch64", - not(any(target_os = "macos", target_os = "linux")), - not(miri), - not(feature = "portable-only") -))] -use core::arch::aarch64::vgetq_lane_u16; #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] use core::arch::aarch64::{ int16x4_t, int16x8_t, int32x4_t, uint16x4_t, uint16x8_t, uint16x8x2_t, vaddq_s16, vaddq_s32, vaddq_u16, vaddq_u32, @@ -35,21 +26,13 @@ use core::arch::aarch64::{ vdup_n_u16, vget_low_u16, vreinterpret_s16_u16, vreinterpret_u16_s16, vreinterpret_u32_u16, vreinterpretq_u16_u32, vreinterpretq_u32_u16, vset_lane_s16, vshr_n_s16, vsub_u16, vuzp1q_u32, vuzp2q_u32, vzip1_u32, vzip2_u32, }; -#[cfg(all( - target_arch = "aarch64", - not(any(target_os = "macos", target_os = "linux")), - not(miri), - not(feature = "portable-only") -))] -use core::arch::aarch64::{vget_high_u8, vget_low_u8, vld3q_u8, vmovl_u8, vorrq_u16, vshlq_n_u16, vshrq_n_u16}; #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] use core::arch::x86_64::{ - __m128i, __m256i, _mm_add_epi16, _mm_and_si128, _mm_cmpgt_epi16, _mm_loadl_epi64, _mm_loadu_si128, _mm_mulhi_epi16, - _mm_mullo_epi16, _mm_set1_epi16, _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8, _mm_srli_epi16, - _mm_storel_epi64, _mm_storeu_si128, _mm_sub_epi16, _mm_unpacklo_epi16, _mm256_add_epi32, _mm256_and_si256, - _mm256_cmpgt_epi32, _mm256_cvtepi16_epi32, _mm256_loadu_si256, _mm256_mulhi_epi16, _mm256_mullo_epi16, - _mm256_mullo_epi32, _mm256_or_si256, _mm256_set1_epi32, _mm256_setzero_si256, _mm256_slli_epi32, _mm256_srai_epi32, - _mm256_srli_epi32, _mm256_storeu_si256, _mm256_sub_epi16, _mm256_sub_epi32, + __m128i, __m256i, _mm_add_epi16, _mm_and_si128, _mm_cmpgt_epi16, _mm_mulhi_epi16, _mm_mullo_epi16, _mm_set1_epi16, + _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8, _mm_srli_epi16, _mm_sub_epi16, _mm_unpacklo_epi16, + _mm256_add_epi32, _mm256_and_si256, _mm256_cmpgt_epi32, _mm256_cvtepi16_epi32, _mm256_mulhi_epi16, + _mm256_mullo_epi16, _mm256_mullo_epi32, _mm256_or_si256, _mm256_set1_epi32, _mm256_setzero_si256, _mm256_slli_epi32, + _mm256_srai_epi32, _mm256_srli_epi32, _mm256_sub_epi16, _mm256_sub_epi32, }; #[cfg(test)] @@ -73,7 +56,7 @@ const Q_U32: u32 = Q as u32; all(target_arch = "aarch64", not(miri), not(feature = "portable-only")), all(target_arch = "x86_64", not(miri), not(feature = "portable-only")) ))] -const Q_I16: i16 = Q as i16; +const Q_I16: i16 = Q.cast_signed(); #[cfg(any( test, miri, @@ -90,7 +73,7 @@ const Q_DIV_RECIP: u64 = 20_642_679; #[cfg(any(test, not(target_arch = "s390x")))] const Q_COMPRESS_DIV_SHIFT: i32 = 33; #[cfg(any(test, not(target_arch = "s390x")))] -const Q_COMPRESS_DIV_RECIP: u64 = 2_580_335; +const Q_COMPRESS_DIV_RECIP: u32 = 2_580_335; const Q_MONT_INV_U16: u16 = 62_209; const MONT_R_SQUARED_MOD_Q: i16 = 1353; #[cfg(test)] @@ -113,6 +96,42 @@ const SAMPLE_NTT_ACC_CHUNK_COEFFS: usize = 16; type Poly = [u16; N]; type PolyVec = [Poly; K]; type PolyMatrix = [PolyVec; K]; + +#[inline(always)] +fn low_byte(value: u16) -> u8 { + value.to_le_bytes()[0] +} + +#[inline(always)] +fn low_u16(value: u32) -> u16 { + let [b0, b1, _, _] = value.to_le_bytes(); + u16::from_le_bytes([b0, b1]) +} + +#[cfg(any(test, not(target_arch = "s390x")))] +#[inline(always)] +fn low_u32(value: u64) -> u32 { + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) +} + +#[cfg(test)] +#[inline] +fn test_low_byte(value: usize) -> u8 { + value.to_le_bytes()[0] +} + +#[cfg(test)] +#[inline] +fn test_u8(value: usize) -> u8 { + u8::try_from(value).expect("test value fits in u8") +} + +#[cfg(test)] +#[inline] +fn test_u16(value: usize) -> u16 { + u16::try_from(value).expect("test value fits in u16") +} #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] type PolyMulCache = [u16; N / 2]; #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] @@ -254,10 +273,7 @@ pub(super) fn validate_and_prepare_decapsulation_key< return Err(MlKemError::InvalidDecapsulationKey); } - let ek = match <&[u8; EK_BYTES]>::try_from(&dk[ek_start..ek_end]) { - Ok(ek) => ek, - Err(_) => unreachable!("ML-KEM decapsulation key layout must include an encapsulation key"), - }; + let ek = <&[u8; EK_BYTES]>::try_from(&dk[ek_start..ek_end]).map_err(|_| MlKemError::InvalidDecapsulationKey)?; Ok(PreparedDecapsulationArithmetic { s_hat: prepare_decapsulation_key::(dk), encapsulation: prepare_encapsulation_key::(ek), @@ -291,7 +307,9 @@ fn prepare_matrix_transpose(rho: &[u8; SEED_BYTES]) -> PolyMatri let mut a_transpose_hat = [[[0u16; N]; K]; K]; for (i, row) in a_transpose_hat.iter_mut().enumerate() { for (j, poly) in row.iter_mut().enumerate() { - sample_ntt_into(rho, i as u8, j as u8, poly); + let i = u8::try_from(i).expect("ML-KEM matrix row fits in u8"); + let j = u8::try_from(j).expect("ML-KEM matrix column fits in u8"); + sample_ntt_into(rho, i, j, poly); } } a_transpose_hat @@ -580,14 +598,11 @@ pub(super) fn decapsulate< >( dk: &[u8; DK_BYTES], c: &[u8; CT_BYTES], -) -> [u8; SHARED_SECRET_BYTES] { +) -> Result<[u8; SHARED_SECRET_BYTES], MlKemError> { let dk_pke = &dk[..DK_PKE_BYTES]; let ek_start = DK_PKE_BYTES; let ek_end = ek_start.strict_add(EK_BYTES); - let ek = match <&[u8; EK_BYTES]>::try_from(&dk[ek_start..ek_end]) { - Ok(ek) => ek, - Err(_) => unreachable!("ML-KEM decapsulation key layout must include an encapsulation key"), - }; + let ek = <&[u8; EK_BYTES]>::try_from(&dk[ek_start..ek_end]).map_err(|_| MlKemError::InvalidDecapsulationKey)?; let h_start = ek_end; let h_stored = &dk[h_start..h_start.strict_add(HASH_BYTES)]; let z = &dk[h_start.strict_add(HASH_BYTES)..]; @@ -624,7 +639,7 @@ pub(super) fn decapsulate< ct::zeroize(&mut c_prime); ct::zeroize(core::slice::from_mut(&mut match_mask)); - shared + Ok(shared) } fn pke_keygen< @@ -686,22 +701,22 @@ fn keygen_sample_noise( let (s1, _) = s_tail.split_at_mut(1); let (e0, e_tail) = t_hat.split_at_mut(1); let (e1, _) = e_tail.split_at_mut(1); - sample_noise_quad::(sigma, 0, &mut s0[0], 1, &mut s1[0], 2, &mut e0[0], 3, &mut e1[0]); + sample_noise_quad::(sigma, [0, 1, 2, 3], [&mut s0[0], &mut s1[0], &mut e0[0], &mut e1[0]]); } else if ETA1_RANDOM_BYTES == ETA2_RANDOM_BYTES && K == 4 { let (s0, s_tail) = s_hat.split_at_mut(1); let (s1, s_tail) = s_tail.split_at_mut(1); let (s2, s3) = s_tail.split_at_mut(1); - sample_noise_quad::(sigma, 0, &mut s0[0], 1, &mut s1[0], 2, &mut s2[0], 3, &mut s3[0]); + sample_noise_quad::(sigma, [0, 1, 2, 3], [&mut s0[0], &mut s1[0], &mut s2[0], &mut s3[0]]); let (e0, e_tail) = t_hat.split_at_mut(1); let (e1, e_tail) = e_tail.split_at_mut(1); let (e2, e3) = e_tail.split_at_mut(1); - sample_noise_quad::(sigma, 4, &mut e0[0], 5, &mut e1[0], 6, &mut e2[0], 7, &mut e3[0]); + sample_noise_quad::(sigma, [4, 5, 6, 7], [&mut e0[0], &mut e1[0], &mut e2[0], &mut e3[0]]); } else if ETA1_RANDOM_BYTES == ETA2_RANDOM_BYTES && K == 3 { let (s0, s_tail) = s_hat.split_at_mut(1); let (s1, s2) = s_tail.split_at_mut(1); let (e0, e_tail) = t_hat.split_at_mut(1); - sample_noise_quad::(sigma, 0, &mut s0[0], 1, &mut s1[0], 2, &mut s2[0], 3, &mut e0[0]); + sample_noise_quad::(sigma, [0, 1, 2, 3], [&mut s0[0], &mut s1[0], &mut s2[0], &mut e0[0]]); let (e1, e2) = e_tail.split_at_mut(1); sample_noise_pair::(sigma, 4, &mut e1[0], 5, &mut e2[0]); @@ -740,14 +755,15 @@ fn keygen_matrix_accumulate(rho: &[u8; SEED_BYTES], s_hat: &Poly #[inline(always)] fn keygen_matrix_accumulate_fused(rho: &[u8; SEED_BYTES], s_hat: &PolyVec, t_hat: &mut PolyVec) { for (i, t_hat_i) in t_hat.iter_mut().enumerate() { + let i_u8 = u8::try_from(i).expect("ML-KEM matrix row fits in u8"); if K == 4 { sample_ntt_quad_mul_accumulate( rho, [ - (0, i as u8, &s_hat[0]), - (1, i as u8, &s_hat[1]), - (2, i as u8, &s_hat[2]), - (3, i as u8, &s_hat[3]), + (0, i_u8, &s_hat[0]), + (1, i_u8, &s_hat[1]), + (2, i_u8, &s_hat[2]), + (3, i_u8, &s_hat[3]), ], t_hat_i, ); @@ -755,17 +771,15 @@ fn keygen_matrix_accumulate_fused(rho: &[u8; SEED_BYTES], s_hat: let mut j = 0usize; while j.strict_add(1) < K { let next = j.strict_add(1); - sample_ntt_pair_mul_accumulate( - rho, - (j as u8, i as u8, &s_hat[j]), - (next as u8, i as u8, &s_hat[next]), - t_hat_i, - ); + let j_u8 = u8::try_from(j).expect("ML-KEM matrix column fits in u8"); + let next_u8 = u8::try_from(next).expect("ML-KEM matrix column fits in u8"); + sample_ntt_pair_mul_accumulate(rho, (j_u8, i_u8, &s_hat[j]), (next_u8, i_u8, &s_hat[next]), t_hat_i); j = j.strict_add(2); } if j < K { - sample_ntt_mul_accumulate(rho, j as u8, i as u8, &s_hat[j], t_hat_i); + let j_u8 = u8::try_from(j).expect("ML-KEM matrix column fits in u8"); + sample_ntt_mul_accumulate(rho, j_u8, i_u8, &s_hat[j], t_hat_i); } } } @@ -1149,7 +1163,8 @@ pub(super) unsafe fn diag_s390x_compress_decompress_values_digest(values: [u16; #[cfg(test)] fn fill_diag_seed(out: &mut [u8; SEED_BYTES], seed: u8) { for (i, byte) in out.iter_mut().enumerate() { - *byte = seed.wrapping_add(i as u8); + let i = u8::try_from(i).expect("ML-KEM diagnostic seed offset fits in u8"); + *byte = seed.wrapping_add(i); } } @@ -1158,7 +1173,8 @@ fn fill_diag_seed(out: &mut [u8; SEED_BYTES], seed: u8) { fn diag_fold_poly(poly: &Poly) -> u16 { let mut acc = 0u16; for (i, &coeff) in poly.iter().enumerate() { - acc ^= coeff.wrapping_mul((i as u16).wrapping_add(1)); + let i = u16::try_from(i).expect("ML-KEM polynomial index fits in u16"); + acc ^= coeff.wrapping_mul(i.wrapping_add(1)); } acc } @@ -1190,7 +1206,7 @@ fn pke_encrypt_prepared_768( let mut e1 = [[0u16; N]; 3]; let [y0, y1, y2] = &mut y_hat; let [e10, e11, e12] = &mut e1; - sample_noise_quad::<128>(r, 0, y0, 1, y1, 2, y2, 3, e10); + sample_noise_quad::<128>(r, [0, 1, 2, 3], [y0, y1, y2, e10]); sample_noise_pair::(r, 4, e11, 5, e12); let mut e2 = [0u16; N]; sample_noise::(r, 6, &mut e2); @@ -1233,9 +1249,9 @@ fn pke_encrypt_prepared_1024( let mut y_hat = [[0u16; N]; 4]; let mut e1 = [[0u16; N]; 4]; let [y0, y1, y2, y3] = &mut y_hat; - sample_noise_quad::<128>(r, 0, y0, 1, y1, 2, y2, 3, y3); + sample_noise_quad::<128>(r, [0, 1, 2, 3], [y0, y1, y2, y3]); let [e10, e11, e12, e13] = &mut e1; - sample_noise_quad::(r, 4, e10, 5, e11, 6, e12, 7, e13); + sample_noise_quad::(r, [4, 5, 6, 7], [e10, e11, e12, e13]); let mut e2 = [0u16; N]; sample_noise::(r, 8, &mut e2); @@ -1324,7 +1340,7 @@ fn pke_encrypt_prepared_768_compare( let mut e1 = [[0u16; N]; 3]; let [y0, y1, y2] = &mut y_hat; let [e10, e11, e12] = &mut e1; - sample_noise_quad::<128>(r, 0, y0, 1, y1, 2, y2, 3, e10); + sample_noise_quad::<128>(r, [0, 1, 2, 3], [y0, y1, y2, e10]); sample_noise_pair::(r, 4, e11, 5, e12); let mut e2 = [0u16; N]; sample_noise::(r, 6, &mut e2); @@ -1368,9 +1384,9 @@ fn pke_encrypt_prepared_1024_compare( let mut y_hat = [[0u16; N]; 4]; let mut e1 = [[0u16; N]; 4]; let [y0, y1, y2, y3] = &mut y_hat; - sample_noise_quad::<128>(r, 0, y0, 1, y1, 2, y2, 3, y3); + sample_noise_quad::<128>(r, [0, 1, 2, 3], [y0, y1, y2, y3]); let [e10, e11, e12, e13] = &mut e1; - sample_noise_quad::(r, 4, e10, 5, e11, 6, e12, 7, e13); + sample_noise_quad::(r, [4, 5, 6, 7], [e10, e11, e12, e13]); let mut e2 = [0u16; N]; sample_noise::(r, 8, &mut e2); @@ -2162,13 +2178,11 @@ fn sample_matrix_ntt_mul_accumulate_materialized_k4_transpose( #[inline] fn matrix_accumulate_coord(entry: usize, transpose: bool) -> ((u8, u8), usize, usize) { - let dst = entry / K; - let rhs = entry % K; - let sample = if transpose { - (dst as u8, rhs as u8) - } else { - (rhs as u8, dst as u8) - }; + let dst = entry.checked_div(K).expect("ML-KEM dimension must be nonzero"); + let rhs = entry.checked_rem(K).expect("ML-KEM dimension must be nonzero"); + let dst_u8 = u8::try_from(dst).expect("ML-KEM matrix row fits in u8"); + let rhs_u8 = u8::try_from(rhs).expect("ML-KEM matrix column fits in u8"); + let sample = if transpose { (dst_u8, rhs_u8) } else { (rhs_u8, dst_u8) }; (sample, dst, rhs) } @@ -2397,7 +2411,7 @@ fn sample_ntt_initial_3blocks_then_tail_into(rho: &[u8; SEED_BYTES], j: u8, i: u let mut filled = 0usize; sample_ntt_initial_3blocks_public(&buf, out, &mut filled); - let mut blocks = SAMPLE_NTT_INITIAL_BLOCKS as u16; + let mut blocks = u16::try_from(SAMPLE_NTT_INITIAL_BLOCKS).expect("initial ML-KEM sample block count fits in u16"); let mut tail = [0u8; SHAKE128_RATE_BYTES]; while filled < N { reader.squeeze(&mut tail); @@ -2537,7 +2551,8 @@ fn sample_ntt_triple_initial_3blocks_then_tail_into( sample_ntt_initial_3blocks_public(&initial[1], out1, &mut filled[1]); sample_ntt_initial_3blocks_public(&initial[2], out2, &mut filled[2]); - let mut blocks = [SAMPLE_NTT_INITIAL_BLOCKS as u16; 3]; + let initial_blocks = u16::try_from(SAMPLE_NTT_INITIAL_BLOCKS).expect("initial ML-KEM sample block count fits in u16"); + let mut blocks = [initial_blocks; 3]; let mut tail = [[0u8; SHAKE128_RATE_BYTES]; 3]; while filled[0] < N && filled[1] < N && filled[2] < N { @@ -2895,125 +2910,6 @@ fn sample_ntt_pair_block_scalar( *filled1 = n1; } -#[cfg(all( - target_arch = "aarch64", - not(any(target_os = "macos", target_os = "linux")), - not(miri), - not(feature = "portable-only") -))] -macro_rules! sample_ntt_extract_16_candidate_vectors_neon { - ($input:expr) => {{ - let mask = vdupq_n_u16(0x0f); - // SAFETY: 48-byte deinterleaved load because: - // 1. The caller guarantees `$input..$input+48` is readable. - // 2. `vld3q_u8` reads exactly three 16-byte vectors from that public SampleNTT byte block. - // 3. The surrounding function is gated by `#[target_feature(enable = "neon")]`. - let triples = unsafe { vld3q_u8($input) }; - - let a0 = vmovl_u8(vget_low_u8(triples.0)); - let a1 = vmovl_u8(vget_low_u8(triples.1)); - let a2 = vmovl_u8(vget_low_u8(triples.2)); - let d0 = vorrq_u16(a0, vshlq_n_u16(vandq_u16(a1, mask), 8)); - let d1 = vorrq_u16(vshrq_n_u16(a1, 4), vshlq_n_u16(a2, 4)); - - let a0 = vmovl_u8(vget_high_u8(triples.0)); - let a1 = vmovl_u8(vget_high_u8(triples.1)); - let a2 = vmovl_u8(vget_high_u8(triples.2)); - let d2 = vorrq_u16(a0, vshlq_n_u16(vandq_u16(a1, mask), 8)); - let d3 = vorrq_u16(vshrq_n_u16(a1, 4), vshlq_n_u16(a2, 4)); - - (d0, d1, d2, d3) - }}; -} - -#[cfg(all( - target_arch = "aarch64", - not(any(target_os = "macos", target_os = "linux")), - not(miri), - not(feature = "portable-only") -))] -macro_rules! sample_ntt_store_public_candidate_unchecked { - ($out:expr, $n:ident, $candidate:expr) => {{ - let candidate = $candidate; - if candidate < Q { - // SAFETY: unchecked public-sample store because: - // 1. The caller reserved enough output capacity for every possible accepted candidate in this - // block, so the current accepted candidate is in bounds. - // 2. `$out` comes from a unique mutable polynomial borrow for the duration of parsing. - // 3. The branch depends only on public matrix-A sample bytes. - unsafe { - *$out.add($n) = candidate; - } - $n = $n.strict_add(1); - } - }}; -} - -#[cfg(all( - target_arch = "aarch64", - not(any(target_os = "macos", target_os = "linux")), - not(miri), - not(feature = "portable-only") -))] -macro_rules! sample_ntt_store_candidate_vectors_neon { - ($out:expr, $n:ident, $candidates:expr) => {{ - let (d0_lo, d1_lo, d0_hi, d1_hi) = $candidates; - macro_rules! store_pair_lane { - ($lane:literal, $d0:expr, $d1:expr) => {{ - sample_ntt_store_public_candidate_unchecked!($out, $n, vgetq_lane_u16::<$lane>($d0)); - sample_ntt_store_public_candidate_unchecked!($out, $n, vgetq_lane_u16::<$lane>($d1)); - }}; - } - - store_pair_lane!(0, d0_lo, d1_lo); - store_pair_lane!(1, d0_lo, d1_lo); - store_pair_lane!(2, d0_lo, d1_lo); - store_pair_lane!(3, d0_lo, d1_lo); - store_pair_lane!(4, d0_lo, d1_lo); - store_pair_lane!(5, d0_lo, d1_lo); - store_pair_lane!(6, d0_lo, d1_lo); - store_pair_lane!(7, d0_lo, d1_lo); - store_pair_lane!(0, d0_hi, d1_hi); - store_pair_lane!(1, d0_hi, d1_hi); - store_pair_lane!(2, d0_hi, d1_hi); - store_pair_lane!(3, d0_hi, d1_hi); - store_pair_lane!(4, d0_hi, d1_hi); - store_pair_lane!(5, d0_hi, d1_hi); - store_pair_lane!(6, d0_hi, d1_hi); - store_pair_lane!(7, d0_hi, d1_hi); - }}; -} - -#[cfg(all( - target_arch = "aarch64", - not(any(target_os = "macos", target_os = "linux")), - not(miri), - not(feature = "portable-only") -))] -#[target_feature(enable = "neon")] -/// # Safety -/// -/// `input` must point to 48 readable bytes. The caller must guarantee that the active CPU supports -/// NEON. `out` must be a unique 32-candidate destination. -unsafe fn sample_ntt_extract_16_candidates_neon(input: *const u8, out: &mut [u16; 32]) { - let (d0_lo, d1_lo, d0_hi, d1_hi) = sample_ntt_extract_16_candidate_vectors_neon!(input); - - // SAFETY: store the first 16 interleaved candidates because: - // 1. `out` is a unique `[u16; 32]` destination. - // 2. `vst2q_u16` writes exactly 16 u16 values from two eight-lane vectors. - // 3. The surrounding function is gated by `#[target_feature(enable = "neon")]`. - unsafe { - vst2q_u16(out.as_mut_ptr(), uint16x8x2_t(d0_lo, d1_lo)); - } - // SAFETY: store the last 16 interleaved candidates because: - // 1. `out.as_mut_ptr().add(16)..add(32)` is inside the 32-candidate destination. - // 2. `vst2q_u16` writes exactly 16 u16 values from two eight-lane vectors. - // 3. The surrounding function is gated by `#[target_feature(enable = "neon")]`. - unsafe { - vst2q_u16(out.as_mut_ptr().add(16), uint16x8x2_t(d0_hi, d1_hi)); - } -} - #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] /// # Safety @@ -3073,105 +2969,33 @@ fn sample_ntt_pair_block_neon( const MAX_CANDIDATES: usize = (SHAKE128_RATE_BYTES / 3) * 2; if N.strict_sub(*filled0) < MAX_CANDIDATES || N.strict_sub(*filled1) < MAX_CANDIDATES { - #[cfg(target_os = "linux")] - { - // SAFETY: bounded aarch64 SampleNTT tail parsing because: - // 1. `buf0` and `buf1` are fixed full SHAKE128 rate blocks. - // 2. The helper caps writes to each polynomial's remaining capacity. - // 3. `out0` and `out1` come from distinct mutable polynomial borrows. - // 4. Rejection branches and write positions depend only on public matrix-A XOF bytes. - unsafe { - sample_ntt_block_asm_bounded(buf0.as_ptr(), out0, filled0); - sample_ntt_block_asm_bounded(buf1.as_ptr(), out1, filled1); - } - } - #[cfg(not(target_os = "linux"))] - { - sample_ntt_block(buf0, out0, filled0); - sample_ntt_block(buf1, out1, filled1); - } - return; - } - - #[cfg(target_os = "linux")] - { - let n0 = *filled0; - let n1 = *filled1; - // SAFETY: aarch64 compact assembly SampleNTT block parsing because: - // 1. `buf0` and `buf1` are full 168-byte SHAKE128 rate blocks. - // 2. The preflight above reserves capacity for all 112 candidates each block can produce. + // SAFETY: bounded aarch64 SampleNTT tail parsing because: + // 1. `buf0` and `buf1` are fixed full SHAKE128 rate blocks. + // 2. The helper caps writes to each polynomial's remaining capacity. // 3. `out0` and `out1` come from distinct mutable polynomial borrows. // 4. Rejection branches and write positions depend only on public matrix-A XOF bytes. - let (count0, count1) = unsafe { - ( - aarch64::sample_ntt_rej_uniform_block_asm(out0.as_mut_ptr().add(n0), buf0.as_ptr()), - aarch64::sample_ntt_rej_uniform_block_asm(out1.as_mut_ptr().add(n1), buf1.as_ptr()), - ) - }; - *filled0 = n0.strict_add(count0); - *filled1 = n1.strict_add(count1); - } - - #[cfg(not(target_os = "linux"))] - { - const NEON_TRIPLES: usize = 16; - const NEON_BYTES: usize = NEON_TRIPLES * 3; - - let mut n0 = *filled0; - let mut n1 = *filled1; - let out0_ptr = out0.as_mut_ptr(); - let out1_ptr = out1.as_mut_ptr(); - let mut offset = 0usize; - - while offset.strict_add(NEON_BYTES) <= SHAKE128_RATE_BYTES { - // SAFETY: fixed 48-byte NEON extraction plus unchecked public-sample stores because: - // 1. `offset + NEON_BYTES <= SHAKE128_RATE_BYTES`, so both inputs name complete chunks. - // 2. The preflight above ensures each output has room for all candidates from the whole rate block. - // 3. Rejection branches and write counts depend only on public matrix-A samples. - let candidates0 = sample_ntt_extract_16_candidate_vectors_neon!(buf0.as_ptr().add(offset)); - let candidates1 = sample_ntt_extract_16_candidate_vectors_neon!(buf1.as_ptr().add(offset)); - sample_ntt_store_candidate_vectors_neon!(out0_ptr, n0, candidates0); - sample_ntt_store_candidate_vectors_neon!(out1_ptr, n1, candidates1); - - offset = offset.strict_add(NEON_BYTES); - } - - while offset.strict_add(2) < SHAKE128_RATE_BYTES { - let a0 = buf0[offset]; - let a1 = buf0[offset.strict_add(1)]; - let a2 = buf0[offset.strict_add(2)]; - let b0 = buf1[offset]; - let b1 = buf1[offset.strict_add(1)]; - let b2 = buf1[offset.strict_add(2)]; - - let d0 = u16::from(a0) | (u16::from(a1 & 0x0f) << 8); - let d1 = (u16::from(a1) >> 4) | (u16::from(a2) << 4); - let e0 = u16::from(b0) | (u16::from(b1 & 0x0f) << 8); - let e1 = (u16::from(b1) >> 4) | (u16::from(b2) << 4); - - if d0 < Q { - out0[n0] = d0; - n0 = n0.strict_add(1); - } - if d1 < Q { - out0[n0] = d1; - n0 = n0.strict_add(1); - } - if e0 < Q { - out1[n1] = e0; - n1 = n1.strict_add(1); - } - if e1 < Q { - out1[n1] = e1; - n1 = n1.strict_add(1); - } - - offset = offset.strict_add(3); + unsafe { + sample_ntt_block_asm_bounded(buf0.as_ptr(), out0, filled0); + sample_ntt_block_asm_bounded(buf1.as_ptr(), out1, filled1); } - - *filled0 = n0; - *filled1 = n1; + return; } + + let n0 = *filled0; + let n1 = *filled1; + // SAFETY: aarch64 compact assembly SampleNTT block parsing because: + // 1. `buf0` and `buf1` are full 168-byte SHAKE128 rate blocks. + // 2. The preflight above reserves capacity for all 112 candidates each block can produce. + // 3. `out0` and `out1` come from distinct mutable polynomial borrows. + // 4. Rejection branches and write positions depend only on public matrix-A XOF bytes. + let (count0, count1) = unsafe { + ( + aarch64::sample_ntt_rej_uniform_block_asm(out0.as_mut_ptr().add(n0), buf0.as_ptr()), + aarch64::sample_ntt_rej_uniform_block_asm(out1.as_mut_ptr().add(n1), buf1.as_ptr()), + ) + }; + *filled0 = n0.strict_add(count0); + *filled1 = n1.strict_add(count1); } fn sample_ntt_triple_from_xof_into(mut readers: [Shake128XofReader; 3], out: [&mut Poly; 3]) { @@ -3250,8 +3074,7 @@ fn sample_ntt_quad_from_xof_into(mut readers: [Shake128XofReader; 4], out: [&mut not(feature = "portable-only") )))] { - let [buf0, buf1, buf2, buf3] = &mut bufs; - Shake128XofReader::squeeze_quad(reader0, reader1, reader2, reader3, buf0, buf1, buf2, buf3); + reader0.squeeze_quad(reader1, reader2, reader3, &mut bufs); sample_ntt_quad_block(&bufs, [out0, out1, out2, out3], &mut filled); } } @@ -3384,7 +3207,8 @@ impl<'a> SampleNttProduct<'a> { #[cfg(not(all(target_arch = "aarch64", not(miri), not(feature = "portable-only"))))] #[inline(always)] fn absorb_block_scalar(&mut self, buf: &[u8; SHAKE128_RATE_BYTES], acc: &mut Poly) { - for chunk in buf.chunks_exact(3) { + let (chunks, remainder) = buf.as_chunks::<3>(); + for chunk in chunks { let d1 = u16::from(chunk[0]) | (u16::from(chunk[1] & 0x0f) << 8); let d2 = (u16::from(chunk[1]) >> 4) | (u16::from(chunk[2]) << 4); @@ -3394,6 +3218,7 @@ impl<'a> SampleNttProduct<'a> { break; } } + debug_assert!(remainder.is_empty()); } #[inline(always)] @@ -3493,6 +3318,9 @@ unsafe fn multiply_ntts_add_assign_chunk_neon_ptr(acc: &mut Poly, a_ptr: *const #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn sample_ntt_product_absorb_block_neon( product: &mut SampleNttProduct<'_>, buf: &[u8; SHAKE128_RATE_BYTES], @@ -3617,6 +3445,10 @@ unsafe fn sample_ntt_product_absorb_rate_ptr_neon( #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1,ssse3")] +/// # Safety +/// +/// The active CPU must support AVX2, SSE4.1, and SSSE3. `buf` must be one complete SHAKE128 rate +/// block, and `product` and `acc` must retain their unique ML-KEM polynomial ownership. fn sample_ntt_product_absorb_block_avx2( product: &mut SampleNttProduct<'_>, buf: &[u8; SHAKE128_RATE_BYTES], @@ -3643,6 +3475,10 @@ fn sample_ntt_product_absorb_block_avx2( #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1,ssse3")] +/// # Safety +/// +/// The active CPU must support AVX2, SSE4.1, and SSSE3. `product` and `acc` must retain their unique +/// ML-KEM polynomial ownership. fn sample_ntt_product_absorb_candidates_avx2( product: &mut SampleNttProduct<'_>, candidates: &[u16; 8], @@ -3755,8 +3591,7 @@ fn sample_ntt_quad_mul_accumulate_from_xof(mut readers: [Shake128XofReader; 4], { while !products[0].is_done() && !products[1].is_done() && !products[2].is_done() && !products[3].is_done() { let [reader0, reader1, reader2, reader3] = &mut readers; - let [buf0, buf1, buf2, buf3] = &mut bufs; - Shake128XofReader::squeeze_quad(reader0, reader1, reader2, reader3, buf0, buf1, buf2, buf3); + reader0.squeeze_quad(reader1, reader2, reader3, &mut bufs); products[0].absorb_block(&bufs[0], acc); products[1].absorb_block(&bufs[1], acc); products[2].absorb_block(&bufs[2], acc); @@ -3997,6 +3832,11 @@ unsafe fn sample_ntt_triple_block_neon_ptrs(rate_ptrs: [*const u8; 3], out: [&mu #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The active CPU must support AArch64 NEON. Every pointer in `rate_ptrs` must +/// reference a readable SHAKE128 rate block. The output polynomials must be +/// distinct, and every value in `filled` must be at most `N`. unsafe fn sample_ntt_quad_block_neon_ptrs(rate_ptrs: [*const u8; 4], out: [&mut Poly; 4], filled: &mut [usize; 4]) { const MAX_CANDIDATES: usize = (SHAKE128_RATE_BYTES / 3) * 2; @@ -4144,6 +3984,10 @@ unsafe fn sample_ntt_block_avx2(buf: &[u8; SHAKE128_RATE_BYTES], out: &mut Poly, #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1,ssse3")] +/// # Safety +/// +/// The active CPU must support AVX2, SSE4.1, and SSSE3. `out` must have room for every accepted +/// candidate in `buf`, as established by the caller's remaining-capacity check. fn sample_ntt_block_avx2_full(buf: &[u8; SHAKE128_RATE_BYTES], out: &mut Poly, filled: &mut usize) { let mut n = *filled; let out_ptr = out.as_mut_ptr(); @@ -4169,6 +4013,10 @@ fn sample_ntt_block_avx2_full(buf: &[u8; SHAKE128_RATE_BYTES], out: &mut Poly, f #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1,ssse3")] +/// # Safety +/// +/// The active CPU must support AVX2, SSE4.1, and SSSE3. `out` and `filled` must describe one unique +/// ML-KEM polynomial destination. fn sample_ntt_block_avx2_bounded(buf: &[u8; SHAKE128_RATE_BYTES], out: &mut Poly, filled: &mut usize) { let mut n = *filled; let out_ptr = out.as_mut_ptr(); @@ -4241,6 +4089,11 @@ fn sample_ntt_store_candidates_bounded(out: *mut u16, n: &mut usize, candidates: #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1,ssse3")] +/// Decode eight SampleNTT candidates from a potentially unaligned 16-byte window. +/// +/// # Safety +/// +/// The active CPU must support AVX2, SSE4.1, and SSSE3, and `input` must be valid to read 16 bytes. unsafe fn sample_ntt_extract_8_candidates_avx2(input: *const u8, shifted_tail: bool) -> [u16; 8] { let pad = i8::MIN; let (even_mask, odd_mask) = if shifted_tail { @@ -4255,24 +4108,17 @@ unsafe fn sample_ntt_extract_8_candidates_avx2(input: *const u8, shifted_tail: b ) }; - // SAFETY: unaligned 16-byte input load because: - // 1. The caller guarantees `input..input + 16` is readable inside the SHAKE128 rate block. - // 2. `_mm_loadu_si128` accepts arbitrary alignment. - // 3. The surrounding function is gated by AVX2/SSE4.1/SSSE3 target features. - let block = unsafe { _mm_loadu_si128(input.cast::<__m128i>()) }; + // SAFETY: the caller guarantees `input..input + 16` is readable inside the SHAKE128 rate block. + let bytes = unsafe { input.cast::<[u8; 16]>().read_unaligned() }; + // SAFETY: `[u8; 16]` and `__m128i` are 128-bit values, and every bit pattern is valid for both. + let block = unsafe { core::mem::transmute::<[u8; 16], __m128i>(bytes) }; let mask_12 = _mm_set1_epi16(0x0fff); let even = _mm_and_si128(_mm_shuffle_epi8(block, even_mask), mask_12); let odd = _mm_srli_epi16::<4>(_mm_shuffle_epi8(block, odd_mask)); let interleaved = _mm_unpacklo_epi16(even, odd); - let mut candidates = [0u16; 8]; - // SAFETY: store exactly eight decoded candidates because: - // 1. `candidates` is a fully initialized local `[u16; 8]` destination. - // 2. `_mm_storeu_si128` writes exactly 16 bytes and accepts arbitrary alignment. - unsafe { - _mm_storeu_si128(candidates.as_mut_ptr().cast::<__m128i>(), interleaved); - } - candidates + // SAFETY: `__m128i` and `[u16; 8]` are 128-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<__m128i, [u16; 8]>(interleaved) } } #[cfg(not(all( @@ -4309,12 +4155,16 @@ fn sample_ntt_block(buf: &[u8; SHAKE128_RATE_BYTES], out: &mut Poly, filled: &mu } fn sample_noise(seed: &[u8; SEED_BYTES], nonce: u8, out: &mut Poly) { + assert!( + matches!(RANDOM_BYTES, ETA2_RANDOM_BYTES | ETA3_RANDOM_BYTES), + "unsupported ML-KEM noise width" + ); let mut buf = [0u8; RANDOM_BYTES]; prf_eta(seed, nonce, &mut buf); match RANDOM_BYTES { ETA2_RANDOM_BYTES => sample_poly_cbd_eta2(&buf, out), ETA3_RANDOM_BYTES => sample_poly_cbd_eta3(&buf, out), - _ => unreachable!("unsupported ML-KEM noise width"), + _ => {} } ct::zeroize(&mut buf); } @@ -4326,6 +4176,10 @@ fn sample_noise_pair( nonce1: u8, out1: &mut Poly, ) { + assert!( + matches!(RANDOM_BYTES, ETA2_RANDOM_BYTES | ETA3_RANDOM_BYTES), + "unsupported ML-KEM noise width" + ); let (mut reader0, mut reader1) = MlKemShake256XofReader::seeded_32_1_pair(seed, nonce0, nonce1); let mut buf0 = [0u8; RANDOM_BYTES]; let mut buf1 = [0u8; RANDOM_BYTES]; @@ -4339,59 +4193,43 @@ fn sample_noise_pair( sample_poly_cbd_eta3(&buf0, out0); sample_poly_cbd_eta3(&buf1, out1); } - _ => unreachable!("unsupported ML-KEM noise width"), + _ => {} } ct::zeroize(&mut buf0); ct::zeroize(&mut buf1); } -#[allow(clippy::too_many_arguments)] fn sample_noise_quad( seed: &[u8; SEED_BYTES], - nonce0: u8, - out0: &mut Poly, - nonce1: u8, - out1: &mut Poly, - nonce2: u8, - out2: &mut Poly, - nonce3: u8, - out3: &mut Poly, + [nonce0, nonce1, nonce2, nonce3]: [u8; 4], + [out0, out1, out2, out3]: [&mut Poly; 4], ) { + assert!( + matches!(RANDOM_BYTES, ETA2_RANDOM_BYTES | ETA3_RANDOM_BYTES), + "unsupported ML-KEM noise width" + ); let (mut reader0, mut reader1, mut reader2, mut reader3) = MlKemShake256XofReader::seeded_32_1_quad(seed, nonce0, nonce1, nonce2, nonce3); - let mut buf0 = [0u8; RANDOM_BYTES]; - let mut buf1 = [0u8; RANDOM_BYTES]; - let mut buf2 = [0u8; RANDOM_BYTES]; - let mut buf3 = [0u8; RANDOM_BYTES]; - MlKemShake256XofReader::squeeze_quad( - &mut reader0, - &mut reader1, - &mut reader2, - &mut reader3, - &mut buf0, - &mut buf1, - &mut buf2, - &mut buf3, - ); + let mut bufs = [[0u8; RANDOM_BYTES]; 4]; + reader0.squeeze_quad(&mut reader1, &mut reader2, &mut reader3, &mut bufs); match RANDOM_BYTES { ETA2_RANDOM_BYTES => { - sample_poly_cbd_eta2(&buf0, out0); - sample_poly_cbd_eta2(&buf1, out1); - sample_poly_cbd_eta2(&buf2, out2); - sample_poly_cbd_eta2(&buf3, out3); + sample_poly_cbd_eta2(&bufs[0], out0); + sample_poly_cbd_eta2(&bufs[1], out1); + sample_poly_cbd_eta2(&bufs[2], out2); + sample_poly_cbd_eta2(&bufs[3], out3); } ETA3_RANDOM_BYTES => { - sample_poly_cbd_eta3(&buf0, out0); - sample_poly_cbd_eta3(&buf1, out1); - sample_poly_cbd_eta3(&buf2, out2); - sample_poly_cbd_eta3(&buf3, out3); + sample_poly_cbd_eta3(&bufs[0], out0); + sample_poly_cbd_eta3(&bufs[1], out1); + sample_poly_cbd_eta3(&bufs[2], out2); + sample_poly_cbd_eta3(&bufs[3], out3); } - _ => unreachable!("unsupported ML-KEM noise width"), + _ => {} + } + for buf in &mut bufs { + ct::zeroize(buf); } - ct::zeroize(&mut buf0); - ct::zeroize(&mut buf1); - ct::zeroize(&mut buf2); - ct::zeroize(&mut buf3); } fn sample_poly_cbd_eta2(input: &[u8], out: &mut Poly) { @@ -4402,26 +4240,31 @@ fn sample_poly_cbd_eta2(input: &[u8], out: &mut Poly) { let y0 = ((byte >> 2) & 1).strict_add((byte >> 3) & 1); let x1 = ((byte >> 4) & 1).strict_add((byte >> 5) & 1); let y1 = ((byte >> 6) & 1).strict_add((byte >> 7) & 1); - out[i.strict_mul(2)] = small_signed_to_mod_q(i16::from(x0) - i16::from(y0)); - out[i.strict_mul(2).strict_add(1)] = small_signed_to_mod_q(i16::from(x1) - i16::from(y1)); + out[i.strict_mul(2)] = small_signed_to_mod_q(i16::from(x0).strict_sub(i16::from(y0))); + out[i.strict_mul(2).strict_add(1)] = small_signed_to_mod_q(i16::from(x1).strict_sub(i16::from(y1))); } } fn sample_poly_cbd_eta3(input: &[u8], out: &mut Poly) { debug_assert_eq!(input.len(), ETA3_RANDOM_BYTES); - for (i, bytes) in input.chunks_exact(3).enumerate() { + let (chunks, remainder) = input.as_chunks::<3>(); + for (i, bytes) in chunks.iter().enumerate() { let bits = u32::from(bytes[0]) | (u32::from(bytes[1]) << 8) | (u32::from(bytes[2]) << 16); - let counts = (bits & 0x0024_9249) + ((bits >> 1) & 0x0024_9249) + ((bits >> 2) & 0x0024_9249); + let counts = (bits & 0x0024_9249) + .strict_add((bits >> 1) & 0x0024_9249) + .strict_add((bits >> 2) & 0x0024_9249); let start = i.strict_mul(4); for j in 0usize..4 { let shift = j.strict_mul(6); let x = (counts >> shift) & 0x7; let y = (counts >> shift.strict_add(3)) & 0x7; - out[start.strict_add(j)] = small_signed_to_mod_q(x as i16 - y as i16); + out[start.strict_add(j)] = + small_signed_to_mod_q(i16::from(low_byte(low_u16(x))).strict_sub(i16::from(low_byte(low_u16(y))))); } } + debug_assert!(remainder.is_empty()); } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] @@ -5116,6 +4959,9 @@ fn multiply_ntts_add_assign_scalar(acc: &mut Poly, a: &Poly, b: &Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn ntt_neon(poly: &mut Poly) { ntt_neon_butterflies(poly); canonicalize_ntt_neon(poly); @@ -5123,6 +4969,9 @@ fn ntt_neon(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn ntt_to_montgomery_product_domain_neon(poly: &mut Poly) { ntt_neon_butterflies(poly); canonicalize_ntt_product_domain_neon(poly); @@ -5130,6 +4979,9 @@ fn ntt_to_montgomery_product_domain_neon(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn ntt_neon_butterflies(poly: &mut Poly) { let mut zeta_index = 1usize; let mut len = 128usize; @@ -5179,6 +5031,9 @@ fn ntt_neon_butterflies(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn ntt_len2_neon(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -5215,6 +5070,9 @@ fn ntt_len2_neon(poly: &mut Poly, zeta_index: &mut usize) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn ntt_len4_neon(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -5244,6 +5102,9 @@ fn ntt_len4_neon(poly: &mut Poly, zeta_index: &mut usize) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn canonicalize_ntt_neon(poly: &mut Poly) { for i in (0..N).step_by(8) { // SAFETY: fixed-size NEON forward-NTT canonicalization because: @@ -5262,6 +5123,9 @@ fn canonicalize_ntt_neon(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn canonicalize_ntt_product_domain_neon(poly: &mut Poly) { for i in (0..N).step_by(8) { // SAFETY: fixed-size NEON forward-NTT product-domain finalization because: @@ -5281,6 +5145,9 @@ fn canonicalize_ntt_product_domain_neon(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn inverse_ntt_neon(poly: &mut Poly, final_scale_mont: i16) { inverse_ntt_neon_butterflies(poly); @@ -5302,6 +5169,9 @@ fn inverse_ntt_neon(poly: &mut Poly, final_scale_mont: i16) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn inverse_ntt_neon_add_assign(poly: &mut Poly, addend: &Poly, final_scale_mont: i16) { inverse_ntt_neon_butterflies(poly); @@ -5323,6 +5193,9 @@ fn inverse_ntt_neon_add_assign(poly: &mut Poly, addend: &Poly, final_scale_mont: #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn inverse_ntt_neon_butterflies(poly: &mut Poly) { let mut zeta_index = 127usize; let mut len = 2usize; @@ -5371,6 +5244,9 @@ fn inverse_ntt_neon_butterflies(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn inverse_ntt_len2_neon(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -5406,6 +5282,9 @@ fn inverse_ntt_len2_neon(poly: &mut Poly, zeta_index: &mut usize) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn inverse_ntt_len4_neon(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -5432,6 +5311,9 @@ fn inverse_ntt_len4_neon(poly: &mut Poly, zeta_index: &mut usize) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn duplicate_i16_pair_lanes_neon(a: i16, b: i16) -> int16x4_t { let lanes = vdup_n_s16(a); let lanes = vset_lane_s16::<2>(b, lanes); @@ -5440,6 +5322,9 @@ fn duplicate_i16_pair_lanes_neon(a: i16, b: i16) -> int16x4_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn zip_u16x4_pair_lanes_neon(lower: uint16x4_t, upper: uint16x4_t) -> uint16x8_t { let lower_pairs: uint32x2_t = vreinterpret_u32_u16(lower); let upper_pairs: uint32x2_t = vreinterpret_u32_u16(upper); @@ -5451,6 +5336,9 @@ fn zip_u16x4_pair_lanes_neon(lower: uint16x4_t, upper: uint16x4_t) -> uint16x8_t #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn poly_mulcache_compute_neon(poly: &Poly, cache: &mut PolyMulCache) { for i in (0..(N / 2)).step_by(8) { let coeff_offset = i.strict_mul(2); @@ -5477,6 +5365,9 @@ fn poly_mulcache_compute_neon(poly: &Poly, cache: &mut PolyMulCache) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn base_multiply_cached_8_neon( a: &Poly, b: &Poly, @@ -5524,6 +5415,9 @@ fn base_multiply_cached_8_neon( #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn multiply_ntts_accumulate_cached_neon( acc: &mut Poly, a: &PolyVec, @@ -5553,6 +5447,9 @@ fn multiply_ntts_accumulate_cached_neon( not(feature = "portable-only") ))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn base_multiply_8_neon(a: &Poly, b: &Poly, gamma: int16x8_t, coeff_offset: usize) -> (uint16x8_t, uint16x8_t) { debug_assert_eq!(coeff_offset % 16, 0); debug_assert!(coeff_offset.strict_add(16) <= N); @@ -5594,6 +5491,9 @@ fn base_multiply_8_neon(a: &Poly, b: &Poly, gamma: int16x8_t, coeff_offset: usiz #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn store_accumulated_8_neon(acc: &mut Poly, coeff_offset: usize, c0: uint16x8_t, c1: uint16x8_t) { debug_assert_eq!(coeff_offset % 16, 0); debug_assert!(coeff_offset.strict_add(16) <= N); @@ -5619,6 +5519,9 @@ fn store_accumulated_8_neon(acc: &mut Poly, coeff_offset: usize, c0: uint16x8_t, not(feature = "portable-only") ))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn multiply_ntts_add_assign_neon(acc: &mut Poly, a: &Poly, b: &Poly) { for i in (0..GAMMAS_MONT.len()).step_by(8) { let coeff_offset = i.strict_mul(2); @@ -5643,6 +5546,9 @@ fn multiply_ntts_add_assign_neon(acc: &mut Poly, a: &Poly, b: &Poly) { not(feature = "portable-only") ))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn multiply_ntts_accumulate_k3_neon(acc: &mut Poly, a: [&Poly; 3], b: [&Poly; 3]) { for i in (0..GAMMAS_MONT.len()).step_by(8) { let coeff_offset = i.strict_mul(2); @@ -5673,6 +5579,9 @@ fn multiply_ntts_accumulate_k3_neon(acc: &mut Poly, a: [&Poly; 3], b: [&Poly; 3] not(feature = "portable-only") ))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn multiply_ntts_accumulate_k4_neon(acc: &mut Poly, a: [&Poly; 4], b: [&Poly; 4]) { for i in (0..GAMMAS_MONT.len()).step_by(8) { let coeff_offset = i.strict_mul(2); @@ -5701,6 +5610,9 @@ fn multiply_ntts_accumulate_k4_neon(acc: &mut Poly, a: [&Poly; 4], b: [&Poly; 4] #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_i16x8_to_i32x4_neon(a: int16x8_t, b: int16x8_t) -> (int32x4_t, int32x4_t) { ( vmull_s16(vget_low_s16(a), vget_low_s16(b)), @@ -5710,38 +5622,56 @@ fn mul_i16x8_to_i32x4_neon(a: int16x8_t, b: int16x8_t) -> (int32x4_t, int32x4_t) #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_mont_const_i16x8(a: int16x8_t, b_mont: i16) -> int16x8_t { montgomery_reduce_s16x8(vmulq_n_s16(a, b_mont), vshrq_n_s16::<1>(vqdmulhq_n_s16(a, b_mont))) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_mont_const_i16x4(a: int16x4_t, b_mont: i16) -> int16x4_t { montgomery_reduce_i32x4_neon(vmull_n_s16(a, b_mont)) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_mont_i16x4(a: int16x4_t, b_mont: int16x4_t) -> int16x4_t { montgomery_reduce_i32x4_neon(vmull_s16(a, b_mont)) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn montgomery_reduce_i32x8_neon(lo: int32x4_t, hi: int32x4_t) -> int16x8_t { vcombine_s16(montgomery_reduce_i32x4_neon(lo), montgomery_reduce_i32x4_neon(hi)) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn montgomery_reduce_i32x4_neon(value: int32x4_t) -> int16x4_t { - let k = vmul_n_s16(vmovn_s32(value), Q_MONT_INV_U16 as i16); + let k = vmul_n_s16(vmovn_s32(value), Q_MONT_INV_U16.cast_signed()); let c = vshrn_n_s32::<16>(vmull_n_s16(k, Q_I16)); vsub_s16(vshrn_n_s32::<16>(value), c) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_mont_const_mod_u16x4(a: uint16x4_t, b_mont: i16) -> uint16x4_t { signed_to_mod_q_s16x4(montgomery_reduce_i32x4_neon(vmull_n_s16( vreinterpret_s16_u16(a), @@ -5751,12 +5681,18 @@ fn mul_mont_const_mod_u16x4(a: uint16x4_t, b_mont: i16) -> uint16x4_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_mont_mod_u16x4(a: uint16x4_t, b_mont: int16x4_t) -> uint16x4_t { signed_to_mod_q_s16x4(montgomery_reduce_i32x4_neon(vmull_s16(vreinterpret_s16_u16(a), b_mont))) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn add_mod_u16x4(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { let sum = vadd_u16(a, b); let q = vdup_n_u16(Q); @@ -5766,6 +5702,9 @@ fn add_mod_u16x4(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn sub_mod_u16x4(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { let diff = vsub_u16(a, b); let q = vdup_n_u16(Q); @@ -5775,6 +5714,9 @@ fn sub_mod_u16x4(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn add_mod_u16x8(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { let sum = vaddq_u16(a, b); let q = vdupq_n_u16(Q); @@ -5784,6 +5726,9 @@ fn add_mod_u16x8(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn sub_mod_u16x8(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { let diff = vsubq_u16(a, b); let q = vdupq_n_u16(Q); @@ -5793,9 +5738,12 @@ fn sub_mod_u16x8(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn canonicalize_lazy_i16x8(value: int16x8_t) -> uint16x8_t { let negative = vshrq_n_s16::<15>(value); - let shifted = vaddq_s16(value, vandq_s16(negative, vdupq_n_s16((Q as i16) * 8))); + let shifted = vaddq_s16(value, vandq_s16(negative, vdupq_n_s16(Q_I16.strict_mul(8)))); let shifted = vreinterpretq_u16_s16(shifted); let lo = reduce_lazy_u32x4(vmovl_u16(vget_low_u16(shifted))); let hi = reduce_lazy_u32x4(vmovl_u16(vget_high_u16(shifted))); @@ -5804,12 +5752,18 @@ fn canonicalize_lazy_i16x8(value: int16x8_t) -> uint16x8_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn lazy_i16x8_to_product_domain(value: int16x8_t) -> uint16x8_t { signed_to_mod_q_s16x8(montgomery_reduce_s16x8(value, vshrq_n_s16::<15>(value))) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn reduce_lazy_u32x4(value: uint32x4_t) -> uint32x4_t { let quotient = vshrq_n_u32::<26>(vaddq_u32(vmulq_n_u32(value, 20_159), vdupq_n_u32(1 << 25))); let product = vmulq_n_u32(quotient, Q_U32); @@ -5820,12 +5774,18 @@ fn reduce_lazy_u32x4(value: uint32x4_t) -> uint32x4_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_mont_const_mod_u16x8(a: uint16x8_t, b_mont: i16) -> uint16x8_t { signed_to_mod_q_s16x8(mul_mont_const_i16x8(vreinterpretq_s16_u16(a), b_mont)) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn montgomery_reduce_s16x8(low: int16x8_t, high: int16x8_t) -> int16x8_t { let k = vreinterpretq_s16_u16(vmulq_n_u16(vreinterpretq_u16_s16(low), Q_MONT_INV_U16)); let c = vshrq_n_s16::<1>(vqdmulhq_n_s16(k, Q_I16)); @@ -5834,6 +5794,9 @@ fn montgomery_reduce_s16x8(low: int16x8_t, high: int16x8_t) -> int16x8_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn signed_to_mod_q_s16x8(value: int16x8_t) -> uint16x8_t { let negative = vshrq_n_s16::<15>(value); vreinterpretq_u16_s16(vaddq_s16(value, vandq_s16(negative, vdupq_n_s16(Q_I16)))) @@ -5841,6 +5804,9 @@ fn signed_to_mod_q_s16x8(value: int16x8_t) -> uint16x8_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn signed_to_mod_q_s16x4(value: int16x4_t) -> uint16x4_t { let negative = vshr_n_s16::<15>(value); vreinterpret_u16_s16(vadd_s16(value, vand_s16(negative, vdup_n_s16(Q_I16)))) @@ -5853,6 +5819,9 @@ fn signed_to_mod_q_s16x4(value: int16x4_t) -> uint16x4_t { not(feature = "portable-only") ))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn to_montgomery_product_domain_neon(poly: &mut Poly) { let high = vdupq_n_s16(0); for i in (0..N).step_by(8) { @@ -5875,6 +5844,9 @@ fn to_montgomery_product_domain_neon(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn from_montgomery_product_domain_neon(poly: &mut Poly) { for i in (0..N).step_by(8) { // SAFETY: fixed-size NEON product-domain exit because: @@ -5897,6 +5869,9 @@ fn from_montgomery_product_domain_neon(poly: &mut Poly) { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn ntt_avx2(poly: &mut Poly) { let mut zeta_index = 1usize; x86_64::ntt_len_ge16_avx2(poly, &mut zeta_index); @@ -5942,6 +5917,9 @@ fn ntt_avx2(poly: &mut Poly) { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn ntt_len4_avx2(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -5965,6 +5943,9 @@ fn ntt_len4_avx2(poly: &mut Poly, zeta_index: &mut usize) { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn inverse_ntt_avx2(poly: &mut Poly, final_scale_mont: i16) { let mut zeta_index = 127usize; let mut len = 2usize; @@ -6029,6 +6010,9 @@ fn inverse_ntt_avx2(poly: &mut Poly, final_scale_mont: i16) { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn inverse_ntt_len4_avx2(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -6055,6 +6039,9 @@ fn inverse_ntt_len4_avx2(poly: &mut Poly, zeta_index: &mut usize) { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn multiply_ntts_add_assign_avx2(acc: &mut Poly, a: &Poly, b: &Poly) { let mask = set1_u32x8_avx2(0xffff); for i in (0..GAMMAS_MONT.len()).step_by(8) { @@ -6069,9 +6056,9 @@ fn multiply_ntts_add_assign_avx2(acc: &mut Poly, a: &Poly, b: &Poly) { // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`, and the caller proves // AVX2 and SSE4.1 availability. unsafe { - let a_pairs = _mm256_loadu_si256(a.as_ptr().add(coeff_offset).cast::<__m256i>()); - let b_pairs = _mm256_loadu_si256(b.as_ptr().add(coeff_offset).cast::<__m256i>()); - let acc_pairs = _mm256_loadu_si256(acc.as_ptr().add(coeff_offset).cast::<__m256i>()); + let a_pairs = load_u16x16_avx2(a.as_ptr().add(coeff_offset)); + let b_pairs = load_u16x16_avx2(b.as_ptr().add(coeff_offset)); + let acc_pairs = load_u16x16_avx2(acc.as_ptr().add(coeff_offset)); let gamma = load_i16x8_as_i32x8_avx2(GAMMAS_MONT.as_ptr().add(i)); let a0 = _mm256_and_si256(a_pairs, mask); @@ -6093,13 +6080,17 @@ fn multiply_ntts_add_assign_avx2(acc: &mut Poly, a: &Poly, b: &Poly) { let out0 = add_mod_u32x8_avx2(acc0, c0); let out1 = add_mod_u32x8_avx2(acc1, c1); let packed = _mm256_or_si256(out0, _mm256_slli_epi32::<16>(out1)); - _mm256_storeu_si256(acc.as_mut_ptr().add(coeff_offset).cast::<__m256i>(), packed); + store_u16x16_avx2(acc.as_mut_ptr().add(coeff_offset), packed); } } } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `coeff_offset..coeff_offset + 16` must be in +/// bounds for `b` and `acc`. fn multiply_ntts_add_assign_chunk_avx2( acc: &mut Poly, a: &[u16; SAMPLE_NTT_ACC_CHUNK_COEFFS], @@ -6118,9 +6109,9 @@ fn multiply_ntts_add_assign_chunk_avx2( // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`, and the caller proves // AVX2 and SSE4.1 availability. unsafe { - let a_pairs = _mm256_loadu_si256(a.as_ptr().cast::<__m256i>()); - let b_pairs = _mm256_loadu_si256(b.as_ptr().add(coeff_offset).cast::<__m256i>()); - let acc_pairs = _mm256_loadu_si256(acc.as_ptr().add(coeff_offset).cast::<__m256i>()); + let a_pairs = load_u16x16_avx2(a.as_ptr()); + let b_pairs = load_u16x16_avx2(b.as_ptr().add(coeff_offset)); + let acc_pairs = load_u16x16_avx2(acc.as_ptr().add(coeff_offset)); let gamma = load_i16x8_as_i32x8_avx2(GAMMAS_MONT.as_ptr().add(gamma_offset)); let a0 = _mm256_and_si256(a_pairs, mask); @@ -6142,60 +6133,109 @@ fn multiply_ntts_add_assign_chunk_avx2( let out0 = add_mod_u32x8_avx2(acc0, c0); let out1 = add_mod_u32x8_avx2(acc1, c1); let packed = _mm256_or_si256(out0, _mm256_slli_epi32::<16>(out1)); - _mm256_storeu_si256(acc.as_mut_ptr().add(coeff_offset).cast::<__m256i>(), packed); + store_u16x16_avx2(acc.as_mut_ptr().add(coeff_offset), packed); } } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn set1_u32x8_avx2(value: u32) -> __m256i { - _mm256_set1_epi32(value as i32) + _mm256_set1_epi32(value.cast_signed()) +} + +#[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] +#[target_feature(enable = "avx2,sse4.1")] +/// Load 16 potentially unaligned coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to read 16 initialized +/// `u16` values. +fn load_u16x16_avx2(ptr: *const u16) -> __m256i { + // SAFETY: the caller provides 16 readable coefficients; `[u16; 16]` retains two-byte alignment. + let lanes = unsafe { ptr.cast::<[u16; 16]>().read_unaligned() }; + // SAFETY: `[u16; 16]` and `__m256i` are 256-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[u16; 16], __m256i>(lanes) } +} + +#[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] +#[target_feature(enable = "avx2,sse4.1")] +/// Store 16 coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to write 16 `u16` values. +fn store_u16x16_avx2(ptr: *mut u16, value: __m256i) { + // SAFETY: `__m256i` and `[u16; 16]` are 256-bit values, and every bit pattern is valid for both. + let lanes = unsafe { core::mem::transmute::<__m256i, [u16; 16]>(value) }; + // SAFETY: the caller provides 16 writable coefficients; `[u16; 16]` retains two-byte alignment. + unsafe { ptr.cast::<[u16; 16]>().write_unaligned(lanes) }; } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to read eight initialized +/// `u16` values. fn load_u16x8_avx2(ptr: *const u16) -> __m128i { - // SAFETY: unaligned 8-coefficient input load because: - // 1. The caller proves `ptr..ptr + 8` is readable. - // 2. `_mm_loadu_si128` accepts arbitrary alignment. - // 3. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`. - unsafe { _mm_loadu_si128(ptr.cast::<__m128i>()) } + // SAFETY: the caller proves `ptr..ptr + 8` is readable. + let lanes = unsafe { ptr.cast::<[u16; 8]>().read_unaligned() }; + // SAFETY: `[u16; 8]` and `__m128i` are 128-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[u16; 8], __m128i>(lanes) } } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to read four initialized +/// `u16` values. fn load_u16x4_avx2(ptr: *const u16) -> __m128i { - // SAFETY: unaligned 4-coefficient input load because: - // 1. The caller proves `ptr..ptr + 4` is readable. - // 2. `_mm_loadl_epi64` accepts arbitrary alignment. - // 3. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`. - unsafe { _mm_loadl_epi64(ptr.cast::<__m128i>()) } + // SAFETY: the caller proves `ptr..ptr + 4` is readable. + let low = unsafe { ptr.cast::<[u16; 4]>().read_unaligned() }; + // Match `_mm_loadl_epi64` by zeroing the upper 64 bits. + let lanes = [low[0], low[1], low[2], low[3], 0, 0, 0, 0]; + // SAFETY: `[u16; 8]` and `__m128i` are 128-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[u16; 8], __m128i>(lanes) } } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to write eight `u16` values. fn store_u16x8_avx2(ptr: *mut u16, values: __m128i) { - // SAFETY: unaligned 8-coefficient output store because: - // 1. The caller proves `ptr..ptr + 8` is writable. - // 2. Values are reduced modulo Q and fit in u16 before storing. - // 3. `_mm_storeu_si128` accepts arbitrary alignment. - // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`. - unsafe { _mm_storeu_si128(ptr.cast::<__m128i>(), values) }; + // SAFETY: `__m128i` and `[u16; 8]` are 128-bit values, and every bit pattern is valid for both. + let lanes = unsafe { core::mem::transmute::<__m128i, [u16; 8]>(values) }; + // SAFETY: the caller proves `ptr..ptr + 8` is writable. + unsafe { ptr.cast::<[u16; 8]>().write_unaligned(lanes) }; } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to write four `u16` values. fn store_u16x4_avx2(ptr: *mut u16, values: __m128i) { - // SAFETY: unaligned 4-coefficient output store because: - // 1. The caller proves `ptr..ptr + 4` is writable. - // 2. Values in the low four lanes are reduced modulo Q and fit in u16 before storing. - // 3. `_mm_storel_epi64` accepts arbitrary alignment. - // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`. - unsafe { _mm_storel_epi64(ptr.cast::<__m128i>(), values) }; + // SAFETY: `__m128i` and `[u16; 8]` are 128-bit values, and every bit pattern is valid for both. + let lanes = unsafe { core::mem::transmute::<__m128i, [u16; 8]>(values) }; + // SAFETY: the caller proves `ptr..ptr + 4` is writable. + unsafe { + ptr + .cast::<[u16; 4]>() + .write_unaligned([lanes[0], lanes[1], lanes[2], lanes[3]]) + }; } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn mul_mont_const_mod_u16x8_avx2(a: __m128i, b_mont: i16) -> __m128i { let b = _mm_set1_epi16(b_mont); signed_to_mod_q_s16x8_avx2(montgomery_reduce_s16x8_avx2( @@ -6206,14 +6246,20 @@ fn mul_mont_const_mod_u16x8_avx2(a: __m128i, b_mont: i16) -> __m128i { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn montgomery_reduce_s16x8_avx2(low: __m128i, high: __m128i) -> __m128i { - let k = _mm_mullo_epi16(low, _mm_set1_epi16(Q_MONT_INV_U16 as i16)); + let k = _mm_mullo_epi16(low, _mm_set1_epi16(Q_MONT_INV_U16.cast_signed())); let c = _mm_mulhi_epi16(k, _mm_set1_epi16(Q_I16)); _mm_sub_epi16(high, c) } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn signed_to_mod_q_s16x8_avx2(value: __m128i) -> __m128i { let negative = _mm_cmpgt_epi16(_mm_setzero_si128(), value); _mm_add_epi16(value, _mm_and_si128(negative, _mm_set1_epi16(Q_I16))) @@ -6221,6 +6267,9 @@ fn signed_to_mod_q_s16x8_avx2(value: __m128i) -> __m128i { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn add_mod_u16x8_avx2(a: __m128i, b: __m128i) -> __m128i { let sum = _mm_add_epi16(a, b); let ge_q = _mm_cmpgt_epi16(sum, _mm_set1_epi16(Q_I16 - 1)); @@ -6229,6 +6278,9 @@ fn add_mod_u16x8_avx2(a: __m128i, b: __m128i) -> __m128i { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn sub_mod_u16x8_avx2(a: __m128i, b: __m128i) -> __m128i { let diff = _mm_sub_epi16(a, b); let borrowed = _mm_cmpgt_epi16(b, a); @@ -6237,17 +6289,27 @@ fn sub_mod_u16x8_avx2(a: __m128i, b: __m128i) -> __m128i { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to read eight initialized +/// `i16` values. fn load_i16x8_as_i32x8_avx2(ptr: *const i16) -> __m256i { // SAFETY: unaligned 8-coefficient AVX2 input load because: // 1. The caller proves `ptr..ptr + 8` is readable. // 2. `_mm_loadu_si128` accepts arbitrary alignment. // 3. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`. - let packed = unsafe { _mm_loadu_si128(ptr.cast::<__m128i>()) }; + // SAFETY: the caller proves `ptr..ptr + 8` is readable. + let lanes = unsafe { ptr.cast::<[i16; 8]>().read_unaligned() }; + // SAFETY: `[i16; 8]` and `__m128i` are 128-bit values, and every bit pattern is valid for both. + let packed = unsafe { core::mem::transmute::<[i16; 8], __m128i>(lanes) }; _mm256_cvtepi16_epi32(packed) } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn montgomery_reduce_i32x8_avx2(value: __m256i) -> __m256i { let k = _mm256_mullo_epi16(value, _mm256_set1_epi32(i32::from(Q_MONT_INV_U16))); let c = _mm256_mulhi_epi16(k, _mm256_set1_epi32(Q_I32)); @@ -6258,6 +6320,9 @@ fn montgomery_reduce_i32x8_avx2(value: __m256i) -> __m256i { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn signed_to_mod_q_i32x8_avx2(value: __m256i) -> __m256i { let negative = _mm256_cmpgt_epi32(_mm256_setzero_si256(), value); _mm256_add_epi32(value, _mm256_and_si256(negative, set1_u32x8_avx2(Q_U32))) @@ -6265,6 +6330,9 @@ fn signed_to_mod_q_i32x8_avx2(value: __m256i) -> __m256i { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn add_mod_u32x8_avx2(a: __m256i, b: __m256i) -> __m256i { let sum = _mm256_add_epi32(a, b); let q = set1_u32x8_avx2(Q_U32); @@ -6284,10 +6352,10 @@ fn base_case_multiply(a0: u16, a1: u16, b0: u16, b1: u16, gamma_mont: i16) -> (u let a0b0 = mul_i32_secret(i32::from(a0), i32::from(b0)); let a1b1 = montgomery_reduce_i32(mul_i32_secret(i32::from(a1), i32::from(b1))); let c0 = signed_to_mod_q(montgomery_reduce_i32( - a0b0 + mul_i32_secret(i32::from(a1b1), i32::from(gamma_mont)), + a0b0.strict_add(mul_i32_secret(i32::from(a1b1), i32::from(gamma_mont))), )); let c1 = signed_to_mod_q(montgomery_reduce_i32( - mul_i32_secret(i32::from(a0), i32::from(b1)) + mul_i32_secret(i32::from(a1), i32::from(b0)), + mul_i32_secret(i32::from(a0), i32::from(b1)).strict_add(mul_i32_secret(i32::from(a1), i32::from(b0))), )); (c0, c1) } @@ -6439,8 +6507,8 @@ fn decompress_poly_add_assign(input: &Poly, out: &mut Poly) { #[inline] fn compress_value(value: u16) -> u16 { - let numerator = (u32::from(value) << D) + Q_HALF; - (div_q_compress_u32(numerator) & ((1u32 << D) - 1)) as u16 + let numerator = (u32::from(value) << D).strict_add(Q_HALF); + low_u16(div_q_compress_u32(numerator) & (1u32 << D).strict_sub(1)) } #[inline(always)] @@ -6493,6 +6561,9 @@ fn compress_values_4(values: [u16; 4]) -> [u16; 4] { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn compress_values_4_neon(values: [u16; 4]) -> [u16; 4] { debug_assert!(matches!(D, 1 | 4 | 5 | 10 | 11)); @@ -6514,18 +6585,25 @@ fn compress_values_4_neon(values: [u16; 4]) -> [u16; 4] { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn compress_lanes_4_neon(lanes: uint16x4_t) -> uint16x4_t { debug_assert!(matches!(D, 1 | 4 | 5 | 10 | 11)); let numerators = vaddq_u32(vshll_n_u16::(lanes), vdupq_n_u32(Q_HALF)); - let recip = vdup_n_u32(Q_COMPRESS_DIV_RECIP as u32); + let recip = vdup_n_u32(Q_COMPRESS_DIV_RECIP); let lo = vshrq_n_u64::(vmull_u32(vget_low_u32(numerators), recip)); let hi = vshrq_n_u64::(vmull_u32(vget_high_u32(numerators), recip)); let quotients = vcombine_u32(vmovn_u64(lo), vmovn_u64(hi)); - vmovn_u32(vandq_u32(quotients, vdupq_n_u32((1u32 << (D as u32)) - 1))) + let mask = 1u32.strict_shl(D.cast_unsigned()).strict_sub(1); + vmovn_u32(vandq_u32(quotients, vdupq_n_u32(mask))) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn load_compress_lanes_4_neon(input: &Poly, offset: usize) -> uint16x4_t { debug_assert!(offset.strict_add(4) <= N); @@ -6540,7 +6618,7 @@ fn load_compress_lanes_4_neon(input: &Poly, offset: usize) -> uint #[inline] fn decompress_value(value: u16) -> u16 { - ((mul_u32_secret(Q_U32, u32::from(value)) + (1u32 << (D - 1))) >> D) as u16 + low_u16(mul_u32_secret(Q_U32, u32::from(value)).strict_add(1u32 << D.strict_sub(1)) >> D) } #[inline(always)] @@ -6592,7 +6670,11 @@ fn decompress_message_add_assign(input: &[u8; SEED_BYTES], out: &mut Poly) { } fn compress_encode_poly(input: &Poly, out: &mut [u8]) { - debug_assert_eq!(out.len(), 32 * D); + assert!( + matches!(D, 1 | 4 | 5 | 10 | 11), + "unsupported ML-KEM fused compress/encode width" + ); + debug_assert_eq!(out.len(), 32usize.strict_mul(D)); match D { 1 => compress_encode_1(input, out), @@ -6600,12 +6682,16 @@ fn compress_encode_poly(input: &Poly, out: &mut [u8]) { 5 => compress_encode_5(input, out), 10 => compress_encode_10(input, out), 11 => compress_encode_11(input, out), - _ => unreachable!("unsupported ML-KEM fused compress/encode width"), + _ => {} } } fn compress_encode_compare_poly(input: &Poly, expected: &[u8]) -> u8 { - debug_assert_eq!(BYTES, 32 * D); + assert!( + matches!(D, 4 | 5 | 10 | 11), + "unsupported ML-KEM fused compress/encode compare width" + ); + debug_assert_eq!(BYTES, 32usize.strict_mul(D)); debug_assert_eq!(expected.len(), BYTES); match D { @@ -6613,7 +6699,7 @@ fn compress_encode_compare_poly(input: &Poly 5 => compress_encode_compare_5(input, expected), 10 => compress_encode_compare_10(input, expected), 11 => compress_encode_compare_11(input, expected), - _ => unreachable!("unsupported ML-KEM fused compress/encode compare width"), + _ => 0, } } @@ -6625,7 +6711,7 @@ fn ct_zero_mask_u8(value: u8) -> u8 { #[inline] fn ct_zero_mask_u64(value: u64) -> u8 { - let nonzero = ((value | value.wrapping_neg()) >> 63) as u8; + let nonzero = ((value | value.wrapping_neg()) >> 63).to_le_bytes()[0]; 0u8.wrapping_sub(nonzero ^ 1) } @@ -6641,8 +6727,8 @@ fn compress_encode_compare_4(input: &Poly, expected: &[u8]) -> u8 { input[j.strict_add(2)], input[j.strict_add(3)], ]); - diff |= ((t[0] | (t[1] << 4)) as u8) ^ expected[k]; - diff |= ((t[2] | (t[3] << 4)) as u8) ^ expected[k.strict_add(1)]; + diff |= low_byte(t[0] | (t[1] << 4)) ^ expected[k]; + diff |= low_byte(t[2] | (t[3] << 4)) ^ expected[k.strict_add(1)]; } ct_zero_mask_u8(diff) } @@ -6668,11 +6754,11 @@ fn compress_encode_compare_5(input: &Poly, expected: &[u8]) -> u8 { let [t0, t1, t2, t3] = lo; let [t4, t5, t6, t7] = hi; - diff |= ((t0 | (t1 << 5)) as u8) ^ expected[k]; - diff |= (((t1 >> 3) | (t2 << 2) | (t3 << 7)) as u8) ^ expected[k.strict_add(1)]; - diff |= (((t3 >> 1) | (t4 << 4)) as u8) ^ expected[k.strict_add(2)]; - diff |= (((t4 >> 4) | (t5 << 1) | (t6 << 6)) as u8) ^ expected[k.strict_add(3)]; - diff |= (((t6 >> 2) | (t7 << 3)) as u8) ^ expected[k.strict_add(4)]; + diff |= low_byte(t0 | (t1 << 5)) ^ expected[k]; + diff |= low_byte((t1 >> 3) | (t2 << 2) | (t3 << 7)) ^ expected[k.strict_add(1)]; + diff |= low_byte((t3 >> 1) | (t4 << 4)) ^ expected[k.strict_add(2)]; + diff |= low_byte((t4 >> 4) | (t5 << 1) | (t6 << 6)) ^ expected[k.strict_add(3)]; + diff |= low_byte((t6 >> 2) | (t7 << 3)) ^ expected[k.strict_add(4)]; } ct_zero_mask_u8(diff) } @@ -6703,11 +6789,11 @@ fn compress_encode_compare_10(input: &Poly, expected: &[u8]) -> u8 { input[j.strict_add(3)], ]); - diff |= (t0 as u8) ^ expected[k]; - diff |= (((t0 >> 8) | (t1 << 2)) as u8) ^ expected[k.strict_add(1)]; - diff |= (((t1 >> 6) | (t2 << 4)) as u8) ^ expected[k.strict_add(2)]; - diff |= (((t2 >> 4) | (t3 << 6)) as u8) ^ expected[k.strict_add(3)]; - diff |= ((t3 >> 2) as u8) ^ expected[k.strict_add(4)]; + diff |= low_byte(t0) ^ expected[k]; + diff |= low_byte((t0 >> 8) | (t1 << 2)) ^ expected[k.strict_add(1)]; + diff |= low_byte((t1 >> 6) | (t2 << 4)) ^ expected[k.strict_add(2)]; + diff |= low_byte((t2 >> 4) | (t3 << 6)) ^ expected[k.strict_add(3)]; + diff |= low_byte(t3 >> 2) ^ expected[k.strict_add(4)]; } ct_zero_mask_u8(diff) } @@ -6734,17 +6820,17 @@ fn compress_encode_compare_11(input: &Poly, expected: &[u8]) -> u8 { let [t0, t1, t2, t3] = lo; let [t4, t5, t6, t7] = hi; - let encoded_lo = u64::from(t0 as u8) - | (u64::from(((t0 >> 8) | (t1 << 3)) as u8) << 8) - | (u64::from(((t1 >> 5) | (t2 << 6)) as u8) << 16) - | (u64::from((t2 >> 2) as u8) << 24) - | (u64::from(((t2 >> 10) | (t3 << 1)) as u8) << 32) - | (u64::from(((t3 >> 7) | (t4 << 4)) as u8) << 40) - | (u64::from(((t4 >> 4) | (t5 << 7)) as u8) << 48) - | (u64::from((t5 >> 1) as u8) << 56); - let encoded_hi = u32::from(((t5 >> 9) | (t6 << 2)) as u8) - | (u32::from(((t6 >> 6) | (t7 << 5)) as u8) << 8) - | (u32::from((t7 >> 3) as u8) << 16); + let encoded_lo = u64::from(low_byte(t0)) + | (u64::from(low_byte((t0 >> 8) | (t1 << 3))) << 8) + | (u64::from(low_byte((t1 >> 5) | (t2 << 6))) << 16) + | (u64::from(low_byte(t2 >> 2)) << 24) + | (u64::from(low_byte((t2 >> 10) | (t3 << 1))) << 32) + | (u64::from(low_byte((t3 >> 7) | (t4 << 4))) << 40) + | (u64::from(low_byte((t4 >> 4) | (t5 << 7))) << 48) + | (u64::from(low_byte(t5 >> 1)) << 56); + let encoded_hi = u32::from(low_byte((t5 >> 9) | (t6 << 2))) + | (u32::from(low_byte((t6 >> 6) | (t7 << 5))) << 8) + | (u32::from(low_byte(t7 >> 3)) << 16); let expected_lo = u64::from_le_bytes([ expected[k], @@ -6768,6 +6854,9 @@ fn compress_encode_compare_11(input: &Poly, expected: &[u8]) -> u8 { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn compress_encode_compare_10_neon(input: &Poly, expected: &[u8]) -> u8 { let mut diff = 0u8; for i in 0usize..64 { @@ -6779,17 +6868,21 @@ fn compress_encode_compare_10_neon(input: &Poly, expected: &[u8]) -> u8 { let t2 = vget_lane_u16::<2>(t); let t3 = vget_lane_u16::<3>(t); - diff |= (t0 as u8) ^ expected[k]; - diff |= (((t0 >> 8) | (t1 << 2)) as u8) ^ expected[k.strict_add(1)]; - diff |= (((t1 >> 6) | (t2 << 4)) as u8) ^ expected[k.strict_add(2)]; - diff |= (((t2 >> 4) | (t3 << 6)) as u8) ^ expected[k.strict_add(3)]; - diff |= ((t3 >> 2) as u8) ^ expected[k.strict_add(4)]; + diff |= low_byte(t0) ^ expected[k]; + diff |= low_byte((t0 >> 8) | (t1 << 2)) ^ expected[k.strict_add(1)]; + diff |= low_byte((t1 >> 6) | (t2 << 4)) ^ expected[k.strict_add(2)]; + diff |= low_byte((t2 >> 4) | (t3 << 6)) ^ expected[k.strict_add(3)]; + diff |= low_byte(t3 >> 2) ^ expected[k.strict_add(4)]; } ct_zero_mask_u8(diff) } fn decode_decompress_poly(input: &[u8], out: &mut Poly) { - debug_assert_eq!(input.len(), 32 * D); + assert!( + matches!(D, 1 | 4 | 5 | 10 | 11), + "unsupported ML-KEM fused decode/decompress width" + ); + debug_assert_eq!(input.len(), 32usize.strict_mul(D)); match D { 1 => decode_decompress_1(input, out), @@ -6797,12 +6890,16 @@ fn decode_decompress_poly(input: &[u8], out: &mut Poly) { 5 => decode_decompress_5(input, out), 10 => decode_decompress_10(input, out), 11 => decode_decompress_11(input, out), - _ => unreachable!("unsupported ML-KEM fused decode/decompress width"), + _ => {} } } fn byte_encode(input: &Poly, out: &mut [u8]) { - debug_assert_eq!(out.len(), 32 * D); + assert!( + matches!(D, 1 | 4 | 5 | 10 | 11 | 12), + "unsupported ML-KEM byte encoding width" + ); + debug_assert_eq!(out.len(), 32usize.strict_mul(D)); match D { 1 => byte_encode_1(input, out), @@ -6811,12 +6908,16 @@ fn byte_encode(input: &Poly, out: &mut [u8]) { 10 => byte_encode_10(input, out), 11 => byte_encode_11(input, out), 12 => byte_encode_12(input, out), - _ => unreachable!("unsupported ML-KEM byte encoding width"), + _ => {} } } fn byte_decode(input: &[u8], out: &mut Poly) { - debug_assert_eq!(input.len(), 32 * D); + assert!( + matches!(D, 1 | 4 | 5 | 10 | 11 | 12), + "unsupported ML-KEM byte decoding width" + ); + debug_assert_eq!(input.len(), 32usize.strict_mul(D)); match D { 1 => byte_decode_1(input, out), @@ -6825,7 +6926,7 @@ fn byte_decode(input: &[u8], out: &mut Poly) { 10 => byte_decode_10(input, out), 11 => byte_decode_11(input, out), 12 => byte_decode_12(input, out), - _ => unreachable!("unsupported ML-KEM byte decoding width"), + _ => {} } } @@ -6844,14 +6945,14 @@ fn compress_encode_1(input: &Poly, out: &mut [u8]) { input[start.strict_add(6)], input[start.strict_add(7)], ]); - *byte = (lo[0] as u8 & 1) - | ((lo[1] as u8 & 1) << 1) - | ((lo[2] as u8 & 1) << 2) - | ((lo[3] as u8 & 1) << 3) - | ((hi[0] as u8 & 1) << 4) - | ((hi[1] as u8 & 1) << 5) - | ((hi[2] as u8 & 1) << 6) - | ((hi[3] as u8 & 1) << 7); + *byte = (low_byte(lo[0]) & 1) + | ((low_byte(lo[1]) & 1) << 1) + | ((low_byte(lo[2]) & 1) << 2) + | ((low_byte(lo[3]) & 1) << 3) + | ((low_byte(hi[0]) & 1) << 4) + | ((low_byte(hi[1]) & 1) << 5) + | ((low_byte(hi[2]) & 1) << 6) + | ((low_byte(hi[3]) & 1) << 7); } } @@ -6870,14 +6971,14 @@ fn subtract_compress_encode_message(lhs: &Poly, rhs: &Poly, out: &mut [u8; SEED_ sub_mod(lhs[start.strict_add(6)], rhs[start.strict_add(6)]), sub_mod(lhs[start.strict_add(7)], rhs[start.strict_add(7)]), ]); - *byte = (lo[0] as u8 & 1) - | ((lo[1] as u8 & 1) << 1) - | ((lo[2] as u8 & 1) << 2) - | ((lo[3] as u8 & 1) << 3) - | ((hi[0] as u8 & 1) << 4) - | ((hi[1] as u8 & 1) << 5) - | ((hi[2] as u8 & 1) << 6) - | ((hi[3] as u8 & 1) << 7); + *byte = (low_byte(lo[0]) & 1) + | ((low_byte(lo[1]) & 1) << 1) + | ((low_byte(lo[2]) & 1) << 2) + | ((low_byte(lo[3]) & 1) << 3) + | ((low_byte(hi[0]) & 1) << 4) + | ((low_byte(hi[1]) & 1) << 5) + | ((low_byte(hi[2]) & 1) << 6) + | ((low_byte(hi[3]) & 1) << 7); } } @@ -6912,7 +7013,7 @@ fn byte_encode_1(input: &Poly, out: &mut [u8]) { let start = i.strict_mul(8); let mut packed = 0u8; for bit in 0..8 { - packed |= ((input[start.strict_add(bit)] & 1) as u8) << bit; + packed |= low_byte(input[start.strict_add(bit)] & 1) << bit; } *byte = packed; } @@ -6937,8 +7038,8 @@ fn compress_encode_4(input: &Poly, out: &mut [u8]) { input[j.strict_add(2)], input[j.strict_add(3)], ]); - out[k] = (t[0] | (t[1] << 4)) as u8; - out[k.strict_add(1)] = (t[2] | (t[3] << 4)) as u8; + out[k] = low_byte(t[0] | (t[1] << 4)); + out[k.strict_add(1)] = low_byte(t[2] | (t[3] << 4)); } } @@ -6964,7 +7065,7 @@ fn decode_decompress_4(input: &[u8], out: &mut Poly) { fn byte_encode_4(input: &Poly, out: &mut [u8]) { for (i, byte) in out.iter_mut().enumerate() { let j = i.strict_mul(2); - *byte = ((input[j] & 0x0f) | ((input[j.strict_add(1)] & 0x0f) << 4)) as u8; + *byte = low_byte((input[j] & 0x0f) | ((input[j.strict_add(1)] & 0x0f) << 4)); } } @@ -6995,11 +7096,11 @@ fn compress_encode_5(input: &Poly, out: &mut [u8]) { let [t0, t1, t2, t3] = lo; let [t4, t5, t6, t7] = hi; - out[k] = (t0 | (t1 << 5)) as u8; - out[k.strict_add(1)] = ((t1 >> 3) | (t2 << 2) | (t3 << 7)) as u8; - out[k.strict_add(2)] = ((t3 >> 1) | (t4 << 4)) as u8; - out[k.strict_add(3)] = ((t4 >> 4) | (t5 << 1) | (t6 << 6)) as u8; - out[k.strict_add(4)] = ((t6 >> 2) | (t7 << 3)) as u8; + out[k] = low_byte(t0 | (t1 << 5)); + out[k.strict_add(1)] = low_byte((t1 >> 3) | (t2 << 2) | (t3 << 7)); + out[k.strict_add(2)] = low_byte((t3 >> 1) | (t4 << 4)); + out[k.strict_add(3)] = low_byte((t4 >> 4) | (t5 << 1) | (t6 << 6)); + out[k.strict_add(4)] = low_byte((t6 >> 2) | (t7 << 3)); } } @@ -7049,11 +7150,11 @@ fn byte_encode_5(input: &Poly, out: &mut [u8]) { let t6 = input[j.strict_add(6)] & 0x001f; let t7 = input[j.strict_add(7)] & 0x001f; - out[k] = (t0 | (t1 << 5)) as u8; - out[k.strict_add(1)] = ((t1 >> 3) | (t2 << 2) | (t3 << 7)) as u8; - out[k.strict_add(2)] = ((t3 >> 1) | (t4 << 4)) as u8; - out[k.strict_add(3)] = ((t4 >> 4) | (t5 << 1) | (t6 << 6)) as u8; - out[k.strict_add(4)] = ((t6 >> 2) | (t7 << 3)) as u8; + out[k] = low_byte(t0 | (t1 << 5)); + out[k.strict_add(1)] = low_byte((t1 >> 3) | (t2 << 2) | (t3 << 7)); + out[k.strict_add(2)] = low_byte((t3 >> 1) | (t4 << 4)); + out[k.strict_add(3)] = low_byte((t4 >> 4) | (t5 << 1) | (t6 << 6)); + out[k.strict_add(4)] = low_byte((t6 >> 2) | (t7 << 3)); } } @@ -7089,11 +7190,11 @@ fn compress_encode_10(input: &Poly, out: &mut [u8]) { input[j.strict_add(3)], ]); - out[k] = t0 as u8; - out[k.strict_add(1)] = ((t0 >> 8) | (t1 << 2)) as u8; - out[k.strict_add(2)] = ((t1 >> 6) | (t2 << 4)) as u8; - out[k.strict_add(3)] = ((t2 >> 4) | (t3 << 6)) as u8; - out[k.strict_add(4)] = (t3 >> 2) as u8; + out[k] = low_byte(t0); + out[k.strict_add(1)] = low_byte((t0 >> 8) | (t1 << 2)); + out[k.strict_add(2)] = low_byte((t1 >> 6) | (t2 << 4)); + out[k.strict_add(3)] = low_byte((t2 >> 4) | (t3 << 6)); + out[k.strict_add(4)] = low_byte(t3 >> 2); } } @@ -7129,11 +7230,11 @@ fn byte_encode_10(input: &Poly, out: &mut [u8]) { let t2 = input[j.strict_add(2)] & 0x03ff; let t3 = input[j.strict_add(3)] & 0x03ff; - out[k] = t0 as u8; - out[k.strict_add(1)] = ((t0 >> 8) | (t1 << 2)) as u8; - out[k.strict_add(2)] = ((t1 >> 6) | (t2 << 4)) as u8; - out[k.strict_add(3)] = ((t2 >> 4) | (t3 << 6)) as u8; - out[k.strict_add(4)] = (t3 >> 2) as u8; + out[k] = low_byte(t0); + out[k.strict_add(1)] = low_byte((t0 >> 8) | (t1 << 2)); + out[k.strict_add(2)] = low_byte((t1 >> 6) | (t2 << 4)); + out[k.strict_add(3)] = low_byte((t2 >> 4) | (t3 << 6)); + out[k.strict_add(4)] = low_byte(t3 >> 2); } } @@ -7173,17 +7274,17 @@ fn compress_encode_11(input: &Poly, out: &mut [u8]) { let [t0, t1, t2, t3] = lo; let [t4, t5, t6, t7] = hi; - out[k] = t0 as u8; - out[k.strict_add(1)] = ((t0 >> 8) | (t1 << 3)) as u8; - out[k.strict_add(2)] = ((t1 >> 5) | (t2 << 6)) as u8; - out[k.strict_add(3)] = (t2 >> 2) as u8; - out[k.strict_add(4)] = ((t2 >> 10) | (t3 << 1)) as u8; - out[k.strict_add(5)] = ((t3 >> 7) | (t4 << 4)) as u8; - out[k.strict_add(6)] = ((t4 >> 4) | (t5 << 7)) as u8; - out[k.strict_add(7)] = (t5 >> 1) as u8; - out[k.strict_add(8)] = ((t5 >> 9) | (t6 << 2)) as u8; - out[k.strict_add(9)] = ((t6 >> 6) | (t7 << 5)) as u8; - out[k.strict_add(10)] = (t7 >> 3) as u8; + out[k] = low_byte(t0); + out[k.strict_add(1)] = low_byte((t0 >> 8) | (t1 << 3)); + out[k.strict_add(2)] = low_byte((t1 >> 5) | (t2 << 6)); + out[k.strict_add(3)] = low_byte(t2 >> 2); + out[k.strict_add(4)] = low_byte((t2 >> 10) | (t3 << 1)); + out[k.strict_add(5)] = low_byte((t3 >> 7) | (t4 << 4)); + out[k.strict_add(6)] = low_byte((t4 >> 4) | (t5 << 7)); + out[k.strict_add(7)] = low_byte(t5 >> 1); + out[k.strict_add(8)] = low_byte((t5 >> 9) | (t6 << 2)); + out[k.strict_add(9)] = low_byte((t6 >> 6) | (t7 << 5)); + out[k.strict_add(10)] = low_byte(t7 >> 3); } } @@ -7239,17 +7340,17 @@ fn byte_encode_11(input: &Poly, out: &mut [u8]) { let t6 = input[j.strict_add(6)] & 0x07ff; let t7 = input[j.strict_add(7)] & 0x07ff; - out[k] = t0 as u8; - out[k.strict_add(1)] = ((t0 >> 8) | (t1 << 3)) as u8; - out[k.strict_add(2)] = ((t1 >> 5) | (t2 << 6)) as u8; - out[k.strict_add(3)] = (t2 >> 2) as u8; - out[k.strict_add(4)] = ((t2 >> 10) | (t3 << 1)) as u8; - out[k.strict_add(5)] = ((t3 >> 7) | (t4 << 4)) as u8; - out[k.strict_add(6)] = ((t4 >> 4) | (t5 << 7)) as u8; - out[k.strict_add(7)] = (t5 >> 1) as u8; - out[k.strict_add(8)] = ((t5 >> 9) | (t6 << 2)) as u8; - out[k.strict_add(9)] = ((t6 >> 6) | (t7 << 5)) as u8; - out[k.strict_add(10)] = (t7 >> 3) as u8; + out[k] = low_byte(t0); + out[k.strict_add(1)] = low_byte((t0 >> 8) | (t1 << 3)); + out[k.strict_add(2)] = low_byte((t1 >> 5) | (t2 << 6)); + out[k.strict_add(3)] = low_byte(t2 >> 2); + out[k.strict_add(4)] = low_byte((t2 >> 10) | (t3 << 1)); + out[k.strict_add(5)] = low_byte((t3 >> 7) | (t4 << 4)); + out[k.strict_add(6)] = low_byte((t4 >> 4) | (t5 << 7)); + out[k.strict_add(7)] = low_byte(t5 >> 1); + out[k.strict_add(8)] = low_byte((t5 >> 9) | (t6 << 2)); + out[k.strict_add(9)] = low_byte((t6 >> 6) | (t7 << 5)); + out[k.strict_add(10)] = low_byte(t7 >> 3); } } @@ -7287,9 +7388,9 @@ fn byte_encode_12(input: &Poly, out: &mut [u8]) { let t0 = input[j]; let t1 = input[j.strict_add(1)]; - out[k] = t0 as u8; - out[k.strict_add(1)] = ((t0 >> 8) | (t1 << 4)) as u8; - out[k.strict_add(2)] = (t1 >> 4) as u8; + out[k] = low_byte(t0); + out[k.strict_add(1)] = low_byte((t0 >> 8) | (t1 << 4)); + out[k.strict_add(2)] = low_byte(t1 >> 4); } } @@ -7308,27 +7409,27 @@ fn byte_decode_12(input: &[u8], out: &mut Poly) { #[inline] fn add_mod(a: u16, b: u16) -> u16 { - let sum = u32::from(a) + u32::from(b); + let sum = u32::from(a).strict_add(u32::from(b)); let reduced = sum.wrapping_sub(Q_U32); - add_q_if_borrowed(reduced) as u16 + low_u16(add_q_if_borrowed(reduced)) } #[inline] fn sub_mod(a: u16, b: u16) -> u16 { let diff = u32::from(a).wrapping_sub(u32::from(b)); - add_q_if_borrowed(diff) as u16 + low_u16(add_q_if_borrowed(diff)) } #[inline] fn sub_if_ge_q(value: u16) -> u16 { let reduced = u32::from(value).wrapping_sub(Q_U32); - add_q_if_borrowed(reduced) as u16 + low_u16(add_q_if_borrowed(reduced)) } #[inline] #[cfg(test)] fn mul_mod(a: u16, b: u16) -> u16 { - reduce_u32(u32::from(a) * u32::from(b)) + reduce_u32(u32::from(a).strict_mul(u32::from(b))) } #[inline] @@ -7376,9 +7477,16 @@ fn from_montgomery_product_domain(value: u16) -> u16 { not(any(target_arch = "aarch64", target_arch = "x86_64")) ))] fn montgomery_reduce_i32(value: i32) -> i16 { - let k = mul_i32_secret(i32::from(value as i16), i32::from(Q_MONT_INV_U16 as i16)); - let c = (mul_i32_secret(i32::from(k as i16), Q_I32) >> 16) as i16; - ((value >> 16) as i16).wrapping_sub(c) + #[inline(always)] + fn low_i16(value: i32) -> i16 { + let [b0, b1, _, _] = value.to_le_bytes(); + i16::from_le_bytes([b0, b1]) + } + + let mont_inverse = i16::from_ne_bytes(Q_MONT_INV_U16.to_ne_bytes()); + let k = mul_i32_secret(i32::from(low_i16(value)), i32::from(mont_inverse)); + let c = low_i16(mul_i32_secret(i32::from(low_i16(k)), Q_I32) >> 16); + low_i16(value >> 16).wrapping_sub(c) } #[inline] @@ -7391,20 +7499,21 @@ fn montgomery_reduce_i32(value: i32) -> i16 { ))] fn signed_to_mod_q(value: i16) -> u16 { let value = i32::from(value); - (value + ((value >> 31) & Q_I32)) as u16 + let reduced = value.strict_add((value >> 31) & Q_I32); + low_u16(u32::from_ne_bytes(reduced.to_ne_bytes())) } #[inline] #[cfg(test)] fn reduce_u32(value: u32) -> u16 { let quotient = div_q_u32(value); - value.wrapping_sub(quotient * Q_U32) as u16 + low_u16(value.wrapping_sub(quotient.strict_mul(Q_U32))) } #[inline] #[cfg(test)] fn div_q_u32(value: u32) -> u32 { - ((u64::from(value) * Q_DIV_RECIP) >> Q_DIV_SHIFT) as u32 + low_u32(u64::from(value).strict_mul(Q_DIV_RECIP) >> Q_DIV_SHIFT) } #[inline] @@ -7415,12 +7524,15 @@ fn div_q_compress_u32(value: u32) -> u32 { } #[cfg(not(target_arch = "s390x"))] { - ((u64::from(value) * Q_COMPRESS_DIV_RECIP) >> Q_COMPRESS_DIV_SHIFT) as u32 + low_u32(u64::from(value).strict_mul(u64::from(Q_COMPRESS_DIV_RECIP)) >> Q_COMPRESS_DIV_SHIFT) } } #[inline] fn mul_u32_secret(a: u32, b: u32) -> u32 { + debug_assert!(u16::try_from(a).is_ok()); + debug_assert!(u16::try_from(b).is_ok()); + #[cfg(target_arch = "s390x")] { // IBM Z integer multiply latency is operand-dependent; keep secret-fed products on a fixed @@ -7429,7 +7541,7 @@ fn mul_u32_secret(a: u32, b: u32) -> u32 { } #[cfg(not(target_arch = "s390x"))] { - a * b + a.wrapping_mul(b) } } @@ -7442,13 +7554,16 @@ fn mul_u32_secret(a: u32, b: u32) -> u32 { not(any(target_arch = "aarch64", target_arch = "x86_64")) ))] fn mul_i32_secret(a: i32, b: i32) -> i32 { + debug_assert!((i32::from(i16::MIN)..=i32::from(i16::MAX)).contains(&a)); + debug_assert!((i32::from(i16::MIN)..=i32::from(i16::MAX)).contains(&b)); + #[cfg(target_arch = "s390x")] { mul_i32_16_ct(a, b) } #[cfg(not(target_arch = "s390x"))] { - a * b + a.wrapping_mul(b) } } @@ -7456,15 +7571,15 @@ fn mul_i32_secret(a: i32, b: i32) -> i32 { #[cfg_attr(not(target_arch = "s390x"), inline)] #[cfg(any(test, target_arch = "s390x"))] fn mul_u32_16_ct(a: u32, b: u32) -> u32 { - debug_assert!(a <= u32::from(u16::MAX)); - debug_assert!(b <= u32::from(u16::MAX)); + debug_assert!(u16::try_from(a).is_ok()); + debug_assert!(u16::try_from(b).is_ok()); let mut acc = 0u32; let mut bit = 0u32; while bit < 16 { let mask = 0u32.wrapping_sub((b >> bit) & 1); acc = acc.wrapping_add((a << bit) & mask); - bit += 1; + bit = bit.strict_add(1); } acc } @@ -7476,13 +7591,15 @@ fn mul_i32_16_ct(a: i32, b: i32) -> i32 { debug_assert!((i32::from(i16::MIN)..=i32::from(i16::MAX)).contains(&a)); debug_assert!((i32::from(i16::MIN)..=i32::from(i16::MAX)).contains(&b)); - let a_sign = (a >> 31) as u32; - let b_sign = (b >> 31) as u32; - let abs_a = ((a as u32) ^ a_sign).wrapping_sub(a_sign); - let abs_b = ((b as u32) ^ b_sign).wrapping_sub(b_sign); + let a_bits = u32::from_ne_bytes(a.to_ne_bytes()); + let b_bits = u32::from_ne_bytes(b.to_ne_bytes()); + let a_sign = 0u32.wrapping_sub(a_bits >> 31); + let b_sign = 0u32.wrapping_sub(b_bits >> 31); + let abs_a = (a_bits ^ a_sign).wrapping_sub(a_sign); + let abs_b = (b_bits ^ b_sign).wrapping_sub(b_sign); let magnitude = mul_u32_16_ct(abs_a, abs_b); let sign = a_sign ^ b_sign; - ((magnitude ^ sign).wrapping_sub(sign)) as i32 + i32::from_ne_bytes((magnitude ^ sign).wrapping_sub(sign).to_ne_bytes()) } #[cfg_attr(target_arch = "s390x", inline(never))] @@ -7495,7 +7612,7 @@ fn div_q_compress_u32_ct(value: u32) -> u32 { let mut remainder = 0u32; let mut bit = 23u32; while bit > 0 { - bit -= 1; + bit = bit.strict_sub(1); remainder = (remainder << 1) | ((value >> bit) & 1); let reduced = remainder.wrapping_sub(Q_U32); let borrow = reduced >> 31; @@ -7529,13 +7646,14 @@ fn opaque_s390x_bit(value: u32) -> u32 { #[inline] #[cfg(test)] fn div_q_compress_u32_recip(value: u32) -> u32 { - ((u64::from(value) * Q_COMPRESS_DIV_RECIP) >> Q_COMPRESS_DIV_SHIFT) as u32 + low_u32(u64::from(value).strict_mul(u64::from(Q_COMPRESS_DIV_RECIP)) >> Q_COMPRESS_DIV_SHIFT) } #[inline] fn small_signed_to_mod_q(value: i16) -> u16 { let value = i32::from(value); - (value + ((value >> 31) & i32::from(Q))) as u16 + let reduced = value.strict_add((value >> 31) & i32::from(Q)); + low_u16(u32::from_ne_bytes(reduced.to_ne_bytes())) } fn h(input: &[u8]) -> [u8; HASH_BYTES] { @@ -7615,11 +7733,11 @@ mod tests { fn miri_mlkem512_portable_round_trip_and_rejection() { let mut key_random = [0u8; MlKem512::KEY_GENERATION_RANDOM_SIZE]; for (i, byte) in key_random.iter_mut().enumerate() { - *byte = (i.strict_mul(29).strict_add(7)) as u8; + *byte = test_low_byte(i.strict_mul(29).strict_add(7)); } let mut encapsulation_random = [0u8; MlKem512::ENCAPSULATION_RANDOM_SIZE]; for (i, byte) in encapsulation_random.iter_mut().enumerate() { - *byte = (i.strict_mul(31).strict_add(11)) as u8; + *byte = test_low_byte(i.strict_mul(31).strict_add(11)); } let (encapsulation_key, decapsulation_key) = MlKem512::generate_keypair(|out| { @@ -7646,7 +7764,7 @@ mod tests { fn ntt_round_trip_preserves_polynomial() { let mut poly = [0u16; N]; for (i, coeff) in poly.iter_mut().enumerate() { - *coeff = ((i.strict_mul(17).strict_add(91)) as u16) % Q; + *coeff = test_u16(i.strict_mul(17).strict_add(91)) % Q; } let original = poly; @@ -7781,22 +7899,22 @@ mod tests { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[test] fn lazy_ntt_neon_finalizers_match_scalar_range() { - let min = -i32::from(Q) * 8; - let max = i32::from(Q) * 8; + let min = -i32::from(Q).strict_mul(8); + let max = i32::from(Q).strict_mul(8); let mut start = min; while start <= max { let lanes = [ start, - (start + 1).min(max), - (start + 2).min(max), - (start + 3).min(max), - (start + 4).min(max), - (start + 5).min(max), - (start + 6).min(max), - (start + 7).min(max), + start.strict_add(1).min(max), + start.strict_add(2).min(max), + start.strict_add(3).min(max), + start.strict_add(4).min(max), + start.strict_add(5).min(max), + start.strict_add(6).min(max), + start.strict_add(7).min(max), ]; - let lanes_i16 = lanes.map(|value| value as i16); + let lanes_i16 = lanes.map(|value| i16::try_from(value).expect("lazy NTT test lane fits in i16")); let mut canonical = [0u16; 8]; let mut product_domain = [0u16; 8]; @@ -7813,9 +7931,9 @@ mod tests { for lane in 0..8 { let mut expected = lanes[lane] % i32::from(Q); if expected < 0 { - expected += i32::from(Q); + expected = expected.strict_add(i32::from(Q)); } - let expected = expected as u16; + let expected = u16::try_from(expected).expect("canonical ML-KEM test coefficient fits in u16"); assert_eq!(canonical[lane], expected, "canonical start {start} lane {lane}"); assert_eq!( product_domain[lane], @@ -7824,7 +7942,7 @@ mod tests { ); } - start += 8; + start = start.strict_add(8); } } @@ -8012,7 +8130,7 @@ mod tests { fn sample_ntt_pair_matches_scalar_samplers() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(37).strict_add(11)) as u8; + *byte = test_low_byte(i.strict_mul(37).strict_add(11)); } let (left, right) = sample_ntt_pair(&rho, 0, 1, 2, 1); @@ -8025,7 +8143,7 @@ mod tests { fn sample_ntt_triple_matches_scalar_samplers() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(43).strict_add(17)) as u8; + *byte = test_low_byte(i.strict_mul(43).strict_add(17)); } let lanes = [(0, 0), (1, 2), (2, 1)]; @@ -8045,11 +8163,13 @@ mod tests { for seed in 0usize..64 { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (seed - .strict_mul(53) - .strict_add(i.strict_mul(47)) - .strict_add((seed >> 1).strict_mul(19)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(53) + .strict_add(i.strict_mul(47)) + .strict_add((seed >> 1).strict_mul(19)) + & 0xff, + ); } for j in 0u8..4 { @@ -8068,7 +8188,7 @@ mod tests { let mut actual = [0u16; N]; let blocks = sample_ntt_initial_3blocks_then_tail_into(&rho, j, i, &mut actual); assert!( - blocks >= SAMPLE_NTT_INITIAL_BLOCKS as u16, + blocks >= test_u16(SAMPLE_NTT_INITIAL_BLOCKS), "seed {seed}, lane ({j}, {i})" ); assert_eq!(actual, expected, "full sample seed {seed}, lane ({j}, {i})"); @@ -8090,11 +8210,13 @@ mod tests { for seed in 0usize..64 { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (seed - .strict_mul(59) - .strict_add(i.strict_mul(31)) - .strict_add((seed >> 2).strict_mul(23)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(59) + .strict_add(i.strict_mul(31)) + .strict_add((seed >> 2).strict_mul(23)) + & 0xff, + ); } for lanes in lane_sets { @@ -8164,11 +8286,13 @@ mod tests { for seed in 0usize..512 { let mut buf = [0u8; SHAKE128_RATE_BYTES]; for (i, byte) in buf.iter_mut().enumerate() { - *byte = (seed - .strict_mul(109) - .strict_add(i.strict_mul(37)) - .strict_add((seed >> 2).strict_mul(41)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(109) + .strict_add(i.strict_mul(37)) + .strict_add((seed >> 2).strict_mul(41)) + & 0xff, + ); } for &start in &fill_offsets { @@ -8206,11 +8330,13 @@ mod tests { for seed in 0usize..512 { let mut buf = [0u8; SHAKE128_RATE_BYTES]; for (i, byte) in buf.iter_mut().enumerate() { - *byte = (seed - .strict_mul(73) - .strict_add(i.strict_mul(29)) - .strict_add((seed >> 1).strict_mul(17)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(73) + .strict_add(i.strict_mul(29)) + .strict_add((seed >> 1).strict_mul(17)) + & 0xff, + ); } let mut expected = [0u16; MAX_CANDIDATES]; @@ -8242,12 +8368,14 @@ mod tests { let mut bufs = [[0u8; SHAKE128_RATE_BYTES]; 3]; for (lane, buf) in bufs.iter_mut().enumerate() { for (i, byte) in buf.iter_mut().enumerate() { - *byte = (seed - .strict_mul(97) - .strict_add(lane.strict_mul(43)) - .strict_add(i.strict_mul(31)) - .strict_add((seed >> 1).strict_mul(19)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(97) + .strict_add(lane.strict_mul(43)) + .strict_add(i.strict_mul(31)) + .strict_add((seed >> 1).strict_mul(19)) + & 0xff, + ); } } @@ -8297,11 +8425,13 @@ mod tests { for seed in 0usize..512 { let mut buf = [0u8; SAMPLE_NTT_INITIAL_BYTES]; for (i, byte) in buf.iter_mut().enumerate() { - *byte = (seed - .strict_mul(83) - .strict_add(i.strict_mul(41)) - .strict_add((seed >> 1).strict_mul(37)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(83) + .strict_add(i.strict_mul(41)) + .strict_add((seed >> 1).strict_mul(37)) + & 0xff, + ); } let mut expected = [0u16; N]; @@ -8336,11 +8466,13 @@ mod tests { for seed in 0usize..256 { let mut buf = [0u8; SHAKE128_RATE_BYTES]; for (i, byte) in buf.iter_mut().enumerate() { - *byte = (seed - .strict_mul(91) - .strict_add(i.strict_mul(19)) - .strict_add((seed >> 2).strict_mul(23)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(91) + .strict_add(i.strict_mul(19)) + .strict_add((seed >> 2).strict_mul(23)) + & 0xff, + ); } for &start in &fill_offsets { @@ -8391,12 +8523,14 @@ mod tests { let mut bufs = [[0u8; SHAKE128_RATE_BYTES]; 3]; for (lane, buf) in bufs.iter_mut().enumerate() { for (i, byte) in buf.iter_mut().enumerate() { - *byte = (seed - .strict_mul(101) - .strict_add(lane.strict_mul(47)) - .strict_add(i.strict_mul(23)) - .strict_add((seed >> 2).strict_mul(29)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(101) + .strict_add(lane.strict_mul(47)) + .strict_add(i.strict_mul(23)) + .strict_add((seed >> 2).strict_mul(29)) + & 0xff, + ); } } @@ -8452,7 +8586,7 @@ mod tests { fn sample_ntt_quad_matches_scalar_samplers() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(41).strict_add(13)) as u8; + *byte = test_low_byte(i.strict_mul(41).strict_add(13)); } let lanes = [(0, 0), (1, 3), (2, 1), (3, 2)]; @@ -8524,7 +8658,7 @@ mod tests { fn seeded_sample_ntt_matches_generic_xof_input() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(31).strict_add(7)) as u8; + *byte = test_low_byte(i.strict_mul(31).strict_add(7)); } for j in 0u8..4 { @@ -8543,7 +8677,7 @@ mod tests { fn seeded_prf_eta_matches_generic_xof_input() { let mut seed = [0u8; SEED_BYTES]; for (i, byte) in seed.iter_mut().enumerate() { - *byte = (i.strict_mul(43).strict_add(5)) as u8; + *byte = test_low_byte(i.strict_mul(43).strict_add(5)); } for nonce in 0u8..8 { @@ -8571,7 +8705,7 @@ mod tests { fn batched_sample_noise_pair_matches_scalar_sampling() { let mut seed = [0u8; SEED_BYTES]; for (i, byte) in seed.iter_mut().enumerate() { - *byte = (i.strict_mul(47).strict_add(13)) as u8; + *byte = test_low_byte(i.strict_mul(47).strict_add(13)); } let mut expected0 = [0u16; N]; @@ -8595,7 +8729,7 @@ mod tests { fn batched_sample_noise_quad_matches_scalar_sampling() { let mut seed = [0u8; SEED_BYTES]; for (i, byte) in seed.iter_mut().enumerate() { - *byte = (i.strict_mul(59).strict_add(19)) as u8; + *byte = test_low_byte(i.strict_mul(59).strict_add(19)); } let mut expected = [[0u16; N]; 4]; @@ -8604,14 +8738,14 @@ mod tests { sample_noise::(&seed, lane.strict_add(3), &mut expected[usize::from(lane)]); } let [actual0, actual1, actual2, actual3] = &mut actual; - sample_noise_quad::(&seed, 3, actual0, 4, actual1, 5, actual2, 6, actual3); + sample_noise_quad::(&seed, [3, 4, 5, 6], [actual0, actual1, actual2, actual3]); assert_eq!(actual, expected, "eta2 quad"); for lane in 0u8..4 { sample_noise::(&seed, lane.strict_add(8), &mut expected[usize::from(lane)]); } let [actual0, actual1, actual2, actual3] = &mut actual; - sample_noise_quad::(&seed, 8, actual0, 9, actual1, 10, actual2, 11, actual3); + sample_noise_quad::(&seed, [8, 9, 10, 11], [actual0, actual1, actual2, actual3]); assert_eq!(actual, expected, "eta3 quad"); } @@ -8619,15 +8753,15 @@ mod tests { fn fused_sample_ntt_accumulate_matches_sample_then_multiply() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(29).strict_add(17)) as u8; + *byte = test_low_byte(i.strict_mul(29).strict_add(17)); } for seed in 0usize..8 { let rhs = test_poly(seed.strict_add(100)); let base = test_poly(seed.strict_add(200)); let mut sampled = [0u16; N]; - let j = (seed % 4) as u8; - let i = ((seed.strict_mul(3)) % 4) as u8; + let j = test_u8(seed % 4); + let i = test_u8(seed.strict_mul(3) % 4); sample_ntt_into(&rho, j, i, &mut sampled); let mut expected = base; @@ -8644,17 +8778,17 @@ mod tests { fn fused_sample_ntt_pair_accumulate_matches_two_sampled_products() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(41).strict_add(23)) as u8; + *byte = test_low_byte(i.strict_mul(41).strict_add(23)); } for seed in 0usize..8 { let rhs0 = test_poly(seed.strict_add(300)); let rhs1 = test_poly(seed.strict_add(400)); let base = test_poly(seed.strict_add(500)); - let j0 = (seed % 4) as u8; - let i0 = ((seed.strict_mul(5).strict_add(1)) % 4) as u8; - let j1 = ((seed.strict_add(2)) % 4) as u8; - let i1 = ((seed.strict_mul(7).strict_add(3)) % 4) as u8; + let j0 = test_u8(seed % 4); + let i0 = test_u8(seed.strict_mul(5).strict_add(1) % 4); + let j1 = test_u8(seed.strict_add(2) % 4); + let i1 = test_u8(seed.strict_mul(7).strict_add(3) % 4); let mut sampled0 = [0u16; N]; let mut sampled1 = [0u16; N]; @@ -8675,7 +8809,7 @@ mod tests { fn fused_sample_ntt_quad_accumulate_matches_four_sampled_products() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(53).strict_add(29)) as u8; + *byte = test_low_byte(i.strict_mul(53).strict_add(29)); } for seed in 0usize..8 { @@ -8685,18 +8819,18 @@ mod tests { let rhs3 = test_poly(seed.strict_add(900)); let rhs = [&rhs0, &rhs1, &rhs2, &rhs3]; let coords = [ - ((seed % 4) as u8, ((seed.strict_mul(3).strict_add(1)) % 4) as u8), + (test_u8(seed % 4), test_u8(seed.strict_mul(3).strict_add(1) % 4)), ( - ((seed.strict_add(1)) % 4) as u8, - ((seed.strict_mul(5).strict_add(2)) % 4) as u8, + test_u8(seed.strict_add(1) % 4), + test_u8(seed.strict_mul(5).strict_add(2) % 4), ), ( - ((seed.strict_add(2)) % 4) as u8, - ((seed.strict_mul(7).strict_add(3)) % 4) as u8, + test_u8(seed.strict_add(2) % 4), + test_u8(seed.strict_mul(7).strict_add(3) % 4), ), ( - ((seed.strict_add(3)) % 4) as u8, - ((seed.strict_mul(11).strict_add(1)) % 4) as u8, + test_u8(seed.strict_add(3) % 4), + test_u8(seed.strict_mul(11).strict_add(1) % 4), ), ]; let base = test_poly(seed.strict_add(1000)); @@ -8728,7 +8862,7 @@ mod tests { fn materialized_k4_matrix_accumulate_matches_reference_layouts() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(61).strict_add(31)) as u8; + *byte = test_low_byte(i.strict_mul(61).strict_add(31)); } let rhs = [test_poly(0x10), test_poly(0x20), test_poly(0x30), test_poly(0x40)]; @@ -8758,7 +8892,7 @@ mod tests { fn materialized_k3_matrix_accumulate_matches_reference_layouts() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(47).strict_add(23)) as u8; + *byte = test_low_byte(i.strict_mul(47).strict_add(23)); } let rhs = [test_poly(0x10), test_poly(0x20), test_poly(0x30)]; @@ -8788,7 +8922,7 @@ mod tests { fn materialized_k2_matrix_accumulate_matches_reference_layouts() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(43).strict_add(19)) as u8; + *byte = test_low_byte(i.strict_mul(43).strict_add(19)); } let rhs = [test_poly(0x10), test_poly(0x20)]; @@ -8814,10 +8948,10 @@ mod tests { let mut rho = [0u8; SEED_BYTES]; let mut sigma = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(37).strict_add(11)) as u8; + *byte = test_low_byte(i.strict_mul(37).strict_add(11)); } for (i, byte) in sigma.iter_mut().enumerate() { - *byte = (i.strict_mul(53).strict_add(19)) as u8; + *byte = test_low_byte(i.strict_mul(53).strict_add(19)); } let mut s_hat = [[0u16; N]; 2]; @@ -8829,7 +8963,7 @@ mod tests { for (i, expected_i) in expected.iter_mut().enumerate() { for (j, rhs) in s_hat.iter().enumerate() { let mut sampled = [0u16; N]; - sample_ntt_into(&rho, j as u8, i as u8, &mut sampled); + sample_ntt_into(&rho, test_u8(j), test_u8(i), &mut sampled); multiply_ntts_add_assign_scalar(expected_i, &sampled, rhs); } } @@ -8850,7 +8984,7 @@ mod tests { fn assert_prepared_matrix_cache_matches_materialized_transpose() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(67).strict_add(29)) as u8; + *byte = test_low_byte(i.strict_mul(67).strict_add(29)); } let mut rhs = [[0u16; N]; K]; @@ -8876,7 +9010,13 @@ mod tests { fn test_poly(seed: usize) -> Poly { let mut poly = [0u16; N]; for (i, coeff) in poly.iter_mut().enumerate() { - *coeff = ((seed.strict_mul(37).strict_add(i.strict_mul(19)).strict_add(11)) % usize::from(Q)) as u16; + let value = seed + .strict_mul(37) + .strict_add(i.strict_mul(19)) + .strict_add(11) + .checked_rem(usize::from(Q)) + .expect("ML-KEM modulus is nonzero"); + *coeff = test_u16(value); } poly } @@ -8941,13 +9081,13 @@ mod tests { let mut m = [0u8; SEED_BYTES]; let mut r = [0u8; SEED_BYTES]; for (i, byte) in key_random.iter_mut().enumerate() { - *byte = (i.strict_mul(13).strict_add(7)) as u8; + *byte = test_low_byte(i.strict_mul(13).strict_add(7)); } for (i, byte) in m.iter_mut().enumerate() { - *byte = (i.strict_mul(17).strict_add(23)) as u8; + *byte = test_low_byte(i.strict_mul(17).strict_add(23)); } for (i, byte) in r.iter_mut().enumerate() { - *byte = (i.strict_mul(19).strict_add(29)) as u8; + *byte = test_low_byte(i.strict_mul(19).strict_add(29)); } let (ek512, _) = keygen::<2, 2, 192, 768, 800, 1632>(&key_random); @@ -8979,7 +9119,7 @@ mod tests { ); for (i, byte) in key_random.iter_mut().enumerate() { - *byte = (i.strict_mul(31).strict_add(11)) as u8; + *byte = test_low_byte(i.strict_mul(31).strict_add(11)); } let (ek1024, _) = keygen::<4, 4, 128, 1536, 1568, 3168>(&key_random); let prepared1024 = prepare_encapsulation_key::<4, 1568>(&ek1024); @@ -9214,9 +9354,9 @@ mod tests { let mut b = [0u16; N]; for i in 0usize..N { - acc[i] = ((seed.strict_mul(19).strict_add(i.strict_mul(7))) % usize::from(Q)) as u16; - a[i] = ((seed.strict_mul(31).strict_add(i.strict_mul(11)).strict_add(5)) % usize::from(Q)) as u16; - b[i] = ((seed.strict_mul(43).strict_add(i.strict_mul(13)).strict_add(17)) % usize::from(Q)) as u16; + acc[i] = test_u16(seed.strict_mul(19).strict_add(i.strict_mul(7)) % usize::from(Q)); + a[i] = test_u16(seed.strict_mul(31).strict_add(i.strict_mul(11)).strict_add(5) % usize::from(Q)); + b[i] = test_u16(seed.strict_mul(43).strict_add(i.strict_mul(13)).strict_add(17) % usize::from(Q)); } let mut scalar = acc; @@ -9378,7 +9518,7 @@ mod tests { fn byte_encode_decode_round_trips_supported_widths() { let mut poly = [0u16; N]; for (i, coeff) in poly.iter_mut().enumerate() { - *coeff = ((i.strict_mul(19).strict_add(7)) as u16) % Q; + *coeff = test_u16(i.strict_mul(19).strict_add(7)) % Q; } let mut encoded_1 = [0u8; 32]; @@ -9455,7 +9595,7 @@ mod tests { fn fused_decode_decompress_matches_two_pass_codec() { let mut input = [0u8; 352]; for (i, byte) in input.iter_mut().enumerate() { - *byte = (i.strict_mul(73).strict_add(19)) as u8; + *byte = test_low_byte(i.strict_mul(73).strict_add(19)); } macro_rules! assert_width { @@ -9482,7 +9622,7 @@ mod tests { fn fused_message_decompress_add_matches_two_pass_codec() { let mut message = [0u8; SEED_BYTES]; for (i, byte) in message.iter_mut().enumerate() { - *byte = (i.strict_mul(37).strict_add(11)) as u8; + *byte = test_low_byte(i.strict_mul(37).strict_add(11)); } let base = test_poly(0x72); @@ -9517,8 +9657,8 @@ mod tests { for d in [1usize, 4, 5, 10, 11] { let max = 1u16 << d; for y in 0..max { - let x = ((Q_U32 * u32::from(y)) + (1u32 << (d - 1))) >> d; - let compressed = (div_q_compress_u32((x << d) + Q_HALF) & ((1u32 << d) - 1)) as u16; + let x = Q_U32.strict_mul(u32::from(y)).strict_add(1u32 << d.strict_sub(1)) >> d; + let compressed = low_u16(div_q_compress_u32((x << d).strict_add(Q_HALF)) & (1u32 << d).strict_sub(1)); assert_eq!(compressed, y, "d={d} y={y}"); } } diff --git a/src/auth/mlkem/portable/aarch64.rs b/src/auth/mlkem/portable/aarch64.rs index 1b007894..3e7f76e8 100644 --- a/src/auth/mlkem/portable/aarch64.rs +++ b/src/auth/mlkem/portable/aarch64.rs @@ -1,6 +1,3 @@ -#![allow(clippy::indexing_slicing)] // Fixed-size ML-KEM native tables and assembly ABI. -#![allow(unsafe_code)] - use core::arch::global_asm; #[cfg(all(test, target_os = "linux"))] @@ -204,6 +201,10 @@ pub(super) unsafe fn sample_ntt_rej_uniform_3blocks_asm(out: *mut u16, input: *c #[inline] #[cfg(any(test, feature = "diag", target_os = "macos"))] +/// # Safety +/// +/// The active platform must provide the matching ML-KEM AArch64 assembly +/// backend. `acc`, `a`, and `b` must be valid, non-overlapping polynomials. pub(super) unsafe fn basemul_accumulate_asm(acc: &mut Poly, a: &Poly, b: &Poly) { #[cfg(target_os = "macos")] { @@ -251,6 +252,11 @@ pub(super) unsafe fn test_basemul_accumulate_asm(acc: &mut Poly, a: &Poly, b: &P #[cfg(any(target_os = "macos", target_os = "linux"))] #[inline] +/// # Safety +/// +/// The active platform must provide the matching ML-KEM AArch64 assembly +/// backend. `a` and `b` must each reference at least `2 * N` readable, +/// contiguous coefficients and must not overlap the uniquely borrowed `acc`. pub(super) unsafe fn basemul_accumulate_k2_asm_ptr(acc: &mut Poly, a: *const u16, b: *const u16) { #[cfg(target_os = "macos")] { @@ -283,6 +289,11 @@ pub(super) unsafe fn basemul_accumulate_k2_asm_ptr(acc: &mut Poly, a: *const u16 #[cfg(any(target_os = "macos", target_os = "linux"))] #[inline] +/// # Safety +/// +/// The active platform must provide the matching ML-KEM AArch64 assembly +/// backend. `a` and `b` must each reference at least `3 * N` readable, +/// contiguous coefficients and must not overlap the uniquely borrowed `acc`. pub(super) unsafe fn basemul_accumulate_k3_asm_ptr(acc: &mut Poly, a: *const u16, b: *const u16) { #[cfg(target_os = "macos")] { @@ -315,6 +326,11 @@ pub(super) unsafe fn basemul_accumulate_k3_asm_ptr(acc: &mut Poly, a: *const u16 #[cfg(any(target_os = "macos", target_os = "linux"))] #[inline] +/// # Safety +/// +/// The active platform must provide the matching ML-KEM AArch64 assembly +/// backend. `a` and `b` must each reference at least `4 * N` readable, +/// contiguous coefficients and must not overlap the uniquely borrowed `acc`. pub(super) unsafe fn basemul_accumulate_k4_asm_ptr(acc: &mut Poly, a: *const u16, b: *const u16) { #[cfg(target_os = "macos")] { diff --git a/src/auth/mlkem/portable/x86_64.rs b/src/auth/mlkem/portable/x86_64.rs index ecbeab37..b599b888 100644 --- a/src/auth/mlkem/portable/x86_64.rs +++ b/src/auth/mlkem/portable/x86_64.rs @@ -1,19 +1,98 @@ -#![allow(clippy::indexing_slicing)] // Fixed-size ML-KEM polynomial kernels. - use core::arch::x86_64::{ - __m128i, __m256i, __m512i, _mm_loadu_si128, _mm256_add_epi16, _mm256_add_epi32, _mm256_and_si256, _mm256_cmpgt_epi16, - _mm256_cmpgt_epi32, _mm256_cvtepi16_epi32, _mm256_loadu_si256, _mm256_mulhi_epi16, _mm256_mullo_epi16, - _mm256_mullo_epi32, _mm256_or_si256, _mm256_permutevar8x32_epi32, _mm256_set_epi16, _mm256_set_epi32, - _mm256_set1_epi16, _mm256_set1_epi32, _mm256_setzero_si256, _mm256_slli_epi32, _mm256_srai_epi32, _mm256_srli_epi32, - _mm256_storeu_si256, _mm256_sub_epi16, _mm256_sub_epi32, _mm256_unpacklo_epi32, _mm512_add_epi32, _mm512_and_si512, - _mm512_cmpgt_epi32_mask, _mm512_cvtepi16_epi32, _mm512_loadu_si512, _mm512_mask_add_epi32, _mm512_mask_sub_epi32, - _mm512_mulhi_epi16, _mm512_mullo_epi16, _mm512_mullo_epi32, _mm512_or_si512, _mm512_set1_epi32, _mm512_setzero_si512, - _mm512_slli_epi32, _mm512_srai_epi32, _mm512_srli_epi32, _mm512_storeu_si512, _mm512_sub_epi16, + __m128i, __m256i, __m512i, _mm256_add_epi16, _mm256_add_epi32, _mm256_and_si256, _mm256_cmpgt_epi16, + _mm256_cmpgt_epi32, _mm256_cvtepi16_epi32, _mm256_mulhi_epi16, _mm256_mullo_epi16, _mm256_mullo_epi32, + _mm256_or_si256, _mm256_permutevar8x32_epi32, _mm256_set_epi16, _mm256_set_epi32, _mm256_set1_epi16, + _mm256_set1_epi32, _mm256_setzero_si256, _mm256_slli_epi32, _mm256_srai_epi32, _mm256_srli_epi32, _mm256_sub_epi16, + _mm256_sub_epi32, _mm256_unpacklo_epi32, _mm512_add_epi32, _mm512_and_si512, _mm512_cmpgt_epi32_mask, + _mm512_cvtepi16_epi32, _mm512_mask_add_epi32, _mm512_mask_sub_epi32, _mm512_mulhi_epi16, _mm512_mullo_epi16, + _mm512_mullo_epi32, _mm512_or_si512, _mm512_set1_epi32, _mm512_setzero_si512, _mm512_slli_epi32, _mm512_srai_epi32, + _mm512_srli_epi32, _mm512_sub_epi16, }; use super::{GAMMAS_MONT, N, Poly, Q_I16, Q_I32, Q_MONT_INV_U16, Q_U32, ZETAS_MONT}; +/// Load 16 potentially unaligned coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 16 initialized `u16` values. +#[inline] +unsafe fn load_u16x16(ptr: *const u16) -> __m256i { + // SAFETY: the caller provides 16 readable coefficients; `[u16; 16]` retains the pointer's two-byte alignment. + let lanes = unsafe { ptr.cast::<[u16; 16]>().read_unaligned() }; + // SAFETY: `[u16; 16]` and `__m256i` are 256-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[u16; 16], __m256i>(lanes) } +} + +/// Store 16 coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to write 16 `u16` values. +#[inline] +unsafe fn store_u16x16(ptr: *mut u16, value: __m256i) { + // SAFETY: `[u16; 16]` and `__m256i` are 256-bit values, and every bit pattern is valid for both. + let lanes = unsafe { core::mem::transmute::<__m256i, [u16; 16]>(value) }; + // SAFETY: the caller provides 16 writable coefficients; `[u16; 16]` retains the pointer's two-byte alignment. + unsafe { ptr.cast::<[u16; 16]>().write_unaligned(lanes) }; +} + +/// Load 32 potentially unaligned coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 32 initialized `u16` values. +#[inline] +unsafe fn load_u16x32(ptr: *const u16) -> __m512i { + // SAFETY: the caller provides 32 readable coefficients; `[u16; 32]` retains the pointer's two-byte alignment. + let lanes = unsafe { ptr.cast::<[u16; 32]>().read_unaligned() }; + // SAFETY: `[u16; 32]` and `__m512i` are 512-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[u16; 32], __m512i>(lanes) } +} + +/// Store 32 coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to write 32 `u16` values. +#[inline] +unsafe fn store_u16x32(ptr: *mut u16, value: __m512i) { + // SAFETY: `[u16; 32]` and `__m512i` are 512-bit values, and every bit pattern is valid for both. + let lanes = unsafe { core::mem::transmute::<__m512i, [u16; 32]>(value) }; + // SAFETY: the caller provides 32 writable coefficients; `[u16; 32]` retains the pointer's two-byte alignment. + unsafe { ptr.cast::<[u16; 32]>().write_unaligned(lanes) }; +} + +/// Load eight potentially unaligned signed coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read eight initialized `i16` values. +#[inline] +unsafe fn load_i16x8(ptr: *const i16) -> __m128i { + // SAFETY: the caller provides eight readable coefficients; `[i16; 8]` retains the pointer's two-byte alignment. + let lanes = unsafe { ptr.cast::<[i16; 8]>().read_unaligned() }; + // SAFETY: `[i16; 8]` and `__m128i` are 128-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[i16; 8], __m128i>(lanes) } +} + +/// Load 16 potentially unaligned signed coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 16 initialized `i16` values. +#[inline] +unsafe fn load_i16x16(ptr: *const i16) -> __m256i { + // SAFETY: the caller provides 16 readable coefficients; `[i16; 16]` retains the pointer's two-byte alignment. + let lanes = unsafe { ptr.cast::<[i16; 16]>().read_unaligned() }; + // SAFETY: `[i16; 16]` and `__m256i` are 256-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[i16; 16], __m256i>(lanes) } +} + #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. pub(super) fn ntt_len_ge16_avx2(poly: &mut Poly, zeta_index: &mut usize) { let mut len = 128usize; while len >= 16 { @@ -32,16 +111,10 @@ pub(super) fn ntt_len_ge16_avx2(poly: &mut Poly, zeta_index: &mut usize) { // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`, and the caller proves // AVX2 and SSE4.1 availability. unsafe { - let u = _mm256_loadu_si256(poly.as_ptr().add(j).cast::<__m256i>()); - let t = mul_mont_mod_u16x16_avx2( - _mm256_loadu_si256(poly.as_ptr().add(j.strict_add(len)).cast::<__m256i>()), - zeta, - ); - _mm256_storeu_si256( - poly.as_mut_ptr().add(j.strict_add(len)).cast::<__m256i>(), - sub_mod_u16x16_avx2(u, t), - ); - _mm256_storeu_si256(poly.as_mut_ptr().add(j).cast::<__m256i>(), add_mod_u16x16_avx2(u, t)); + let u = load_u16x16(poly.as_ptr().add(j)); + let t = mul_mont_mod_u16x16_avx2(load_u16x16(poly.as_ptr().add(j.strict_add(len))), zeta); + store_u16x16(poly.as_mut_ptr().add(j.strict_add(len)), sub_mod_u16x16_avx2(u, t)); + store_u16x16(poly.as_mut_ptr().add(j), add_mod_u16x16_avx2(u, t)); } j = j.strict_add(16); } @@ -52,6 +125,9 @@ pub(super) fn ntt_len_ge16_avx2(poly: &mut Poly, zeta_index: &mut usize) { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. pub(super) fn inverse_ntt_len_ge16_avx2(poly: &mut Poly, zeta_index: &mut usize) { let mut len = 16usize; while len <= 128 { @@ -70,11 +146,11 @@ pub(super) fn inverse_ntt_len_ge16_avx2(poly: &mut Poly, zeta_index: &mut usize) // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`, and the caller proves // AVX2 and SSE4.1 availability. unsafe { - let t = _mm256_loadu_si256(poly.as_ptr().add(j).cast::<__m256i>()); - let u = _mm256_loadu_si256(poly.as_ptr().add(j.strict_add(len)).cast::<__m256i>()); - _mm256_storeu_si256(poly.as_mut_ptr().add(j).cast::<__m256i>(), add_mod_u16x16_avx2(t, u)); - _mm256_storeu_si256( - poly.as_mut_ptr().add(j.strict_add(len)).cast::<__m256i>(), + let t = load_u16x16(poly.as_ptr().add(j)); + let u = load_u16x16(poly.as_ptr().add(j.strict_add(len))); + store_u16x16(poly.as_mut_ptr().add(j), add_mod_u16x16_avx2(t, u)); + store_u16x16( + poly.as_mut_ptr().add(j.strict_add(len)), mul_mont_mod_u16x16_avx2(sub_mod_u16x16_avx2(u, t), zeta), ); } @@ -87,6 +163,9 @@ pub(super) fn inverse_ntt_len_ge16_avx2(poly: &mut Poly, zeta_index: &mut usize) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. pub(super) fn ntt_len2_avx2(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -103,15 +182,15 @@ pub(super) fn ntt_len2_avx2(poly: &mut Poly, zeta_index: &mut usize) { // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`, and the caller proves // AVX2 and SSE4.1 availability. unsafe { - let values = _mm256_loadu_si256(poly.as_ptr().add(start).cast::<__m256i>()); + let values = load_u16x16(poly.as_ptr().add(start)); let lower = deinterleave_len2_lower_avx2(values); let upper = deinterleave_len2_upper_avx2(values); let twiddles = duplicate_i16_quartet_lanes_avx2(zeta0, zeta1, zeta2, zeta3); let t = mul_mont_mod_u16x16_avx2(upper, twiddles); let lower_out = add_mod_u16x16_avx2(lower, t); let upper_out = sub_mod_u16x16_avx2(lower, t); - _mm256_storeu_si256( - poly.as_mut_ptr().add(start).cast::<__m256i>(), + store_u16x16( + poly.as_mut_ptr().add(start), _mm256_unpacklo_epi32(lower_out, upper_out), ); } @@ -121,6 +200,9 @@ pub(super) fn ntt_len2_avx2(poly: &mut Poly, zeta_index: &mut usize) { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. pub(super) fn inverse_ntt_len2_avx2(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -140,14 +222,14 @@ pub(super) fn inverse_ntt_len2_avx2(poly: &mut Poly, zeta_index: &mut usize) { // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`, and the caller proves // AVX2 and SSE4.1 availability. unsafe { - let values = _mm256_loadu_si256(poly.as_ptr().add(start).cast::<__m256i>()); + let values = load_u16x16(poly.as_ptr().add(start)); let lower = deinterleave_len2_lower_avx2(values); let upper = deinterleave_len2_upper_avx2(values); let twiddles = duplicate_i16_quartet_lanes_avx2(zeta0, zeta1, zeta2, zeta3); let lower_out = add_mod_u16x16_avx2(lower, upper); let upper_out = mul_mont_mod_u16x16_avx2(sub_mod_u16x16_avx2(upper, lower), twiddles); - _mm256_storeu_si256( - poly.as_mut_ptr().add(start).cast::<__m256i>(), + store_u16x16( + poly.as_mut_ptr().add(start), _mm256_unpacklo_epi32(lower_out, upper_out), ); } @@ -157,6 +239,9 @@ pub(super) fn inverse_ntt_len2_avx2(poly: &mut Poly, zeta_index: &mut usize) { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. pub(super) fn multiply_ntts_accumulate_k3_avx2(acc: &mut Poly, a: [&Poly; 3], b: [&Poly; 3]) { for i in (0..GAMMAS_MONT.len()).step_by(8) { let coeff_offset = i.strict_mul(2); @@ -170,7 +255,7 @@ pub(super) fn multiply_ntts_accumulate_k3_avx2(acc: &mut Poly, a: [&Poly; 3], b: // AVX2 and SSE4.1 availability. unsafe { let gamma = load_i16x8_as_i32x8_avx2(GAMMAS_MONT.as_ptr().add(i)); - let acc_pairs = _mm256_loadu_si256(acc.as_ptr().add(coeff_offset).cast::<__m256i>()); + let acc_pairs = load_u16x16(acc.as_ptr().add(coeff_offset)); let (p00, p01) = base_multiply_chunk_avx2(a[0], b[0], gamma, coeff_offset); let (p10, p11) = base_multiply_chunk_avx2(a[1], b[1], gamma, coeff_offset); let (p20, p21) = base_multiply_chunk_avx2(a[2], b[2], gamma, coeff_offset); @@ -189,6 +274,9 @@ pub(super) fn multiply_ntts_accumulate_k3_avx2(acc: &mut Poly, a: [&Poly; 3], b: } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. pub(super) fn multiply_ntts_accumulate_k4_avx2(acc: &mut Poly, a: [&Poly; 4], b: [&Poly; 4]) { for i in (0..GAMMAS_MONT.len()).step_by(8) { let coeff_offset = i.strict_mul(2); @@ -202,7 +290,7 @@ pub(super) fn multiply_ntts_accumulate_k4_avx2(acc: &mut Poly, a: [&Poly; 4], b: // AVX2 and SSE4.1 availability. unsafe { let gamma = load_i16x8_as_i32x8_avx2(GAMMAS_MONT.as_ptr().add(i)); - let acc_pairs = _mm256_loadu_si256(acc.as_ptr().add(coeff_offset).cast::<__m256i>()); + let acc_pairs = load_u16x16(acc.as_ptr().add(coeff_offset)); let (p00, p01) = base_multiply_chunk_avx2(a[0], b[0], gamma, coeff_offset); let (p10, p11) = base_multiply_chunk_avx2(a[1], b[1], gamma, coeff_offset); let (p20, p21) = base_multiply_chunk_avx2(a[2], b[2], gamma, coeff_offset); @@ -228,6 +316,9 @@ pub(super) fn multiply_ntts_accumulate_k4_avx2(acc: &mut Poly, a: [&Poly; 4], b: } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. pub(super) fn multiply_ntts_accumulate_k3_avx512(acc: &mut Poly, a: [&Poly; 3], b: [&Poly; 3]) { for i in (0..GAMMAS_MONT.len()).step_by(16) { let coeff_offset = i.strict_mul(2); @@ -242,7 +333,7 @@ pub(super) fn multiply_ntts_accumulate_k3_avx512(acc: &mut Poly, a: [&Poly; 3], // available. unsafe { let gamma = load_i16x16_as_i32x16_avx512(GAMMAS_MONT.as_ptr().add(i)); - let acc_pairs = _mm512_loadu_si512(acc.as_ptr().add(coeff_offset).cast::<__m512i>()); + let acc_pairs = load_u16x32(acc.as_ptr().add(coeff_offset)); let (p00, p01) = base_multiply_chunk_avx512(a[0], b[0], gamma, coeff_offset); let (p10, p11) = base_multiply_chunk_avx512(a[1], b[1], gamma, coeff_offset); let (p20, p21) = base_multiply_chunk_avx512(a[2], b[2], gamma, coeff_offset); @@ -261,6 +352,9 @@ pub(super) fn multiply_ntts_accumulate_k3_avx512(acc: &mut Poly, a: [&Poly; 3], } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. pub(super) fn multiply_ntts_accumulate_k4_avx512(acc: &mut Poly, a: [&Poly; 4], b: [&Poly; 4]) { for i in (0..GAMMAS_MONT.len()).step_by(16) { let coeff_offset = i.strict_mul(2); @@ -275,7 +369,7 @@ pub(super) fn multiply_ntts_accumulate_k4_avx512(acc: &mut Poly, a: [&Poly; 4], // available. unsafe { let gamma = load_i16x16_as_i32x16_avx512(GAMMAS_MONT.as_ptr().add(i)); - let acc_pairs = _mm512_loadu_si512(acc.as_ptr().add(coeff_offset).cast::<__m512i>()); + let acc_pairs = load_u16x32(acc.as_ptr().add(coeff_offset)); let (p00, p01) = base_multiply_chunk_avx512(a[0], b[0], gamma, coeff_offset); let (p10, p11) = base_multiply_chunk_avx512(a[1], b[1], gamma, coeff_offset); let (p20, p21) = base_multiply_chunk_avx512(a[2], b[2], gamma, coeff_offset); @@ -301,6 +395,9 @@ pub(super) fn multiply_ntts_accumulate_k4_avx512(acc: &mut Poly, a: [&Poly; 4], } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. pub(super) fn multiply_ntts_add_assign_avx512(acc: &mut Poly, a: &Poly, b: &Poly) { for i in (0..GAMMAS_MONT.len()).step_by(16) { let coeff_offset = i.strict_mul(2); @@ -315,7 +412,7 @@ pub(super) fn multiply_ntts_add_assign_avx512(acc: &mut Poly, a: &Poly, b: &Poly // available. unsafe { let gamma = load_i16x16_as_i32x16_avx512(GAMMAS_MONT.as_ptr().add(i)); - let acc_pairs = _mm512_loadu_si512(acc.as_ptr().add(coeff_offset).cast::<__m512i>()); + let acc_pairs = load_u16x32(acc.as_ptr().add(coeff_offset)); let (product0, product1) = base_multiply_chunk_avx512(a, b, gamma, coeff_offset); let out0 = add_mod_u32x16_avx512(acc_lanes0_avx512(acc_pairs), product0); @@ -326,21 +423,33 @@ pub(super) fn multiply_ntts_add_assign_avx512(acc: &mut Poly, a: &Poly, b: &Poly } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn deinterleave_len2_lower_avx2(values: __m256i) -> __m256i { _mm256_permutevar8x32_epi32(values, _mm256_set_epi32(0, 0, 6, 4, 0, 0, 2, 0)) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn deinterleave_len2_upper_avx2(values: __m256i) -> __m256i { _mm256_permutevar8x32_epi32(values, _mm256_set_epi32(0, 0, 7, 5, 0, 0, 3, 1)) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn duplicate_i16_quartet_lanes_avx2(a: i16, b: i16, c: i16, d: i16) -> __m256i { _mm256_set_epi16(0, 0, 0, 0, d, d, c, c, 0, 0, 0, 0, b, b, a, a) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn mul_mont_mod_u16x16_avx2(a: __m256i, b_mont: __m256i) -> __m256i { signed_to_mod_q_s16x16_avx2(montgomery_reduce_s16x16_avx2( _mm256_mullo_epi16(a, b_mont), @@ -349,19 +458,28 @@ fn mul_mont_mod_u16x16_avx2(a: __m256i, b_mont: __m256i) -> __m256i { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn montgomery_reduce_s16x16_avx2(low: __m256i, high: __m256i) -> __m256i { - let k = _mm256_mullo_epi16(low, _mm256_set1_epi16(Q_MONT_INV_U16 as i16)); + let k = _mm256_mullo_epi16(low, _mm256_set1_epi16(Q_MONT_INV_U16.cast_signed())); let c = _mm256_mulhi_epi16(k, _mm256_set1_epi16(Q_I16)); _mm256_sub_epi16(high, c) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn signed_to_mod_q_s16x16_avx2(value: __m256i) -> __m256i { let negative = _mm256_cmpgt_epi16(_mm256_setzero_si256(), value); _mm256_add_epi16(value, _mm256_and_si256(negative, _mm256_set1_epi16(Q_I16))) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn add_mod_u16x16_avx2(a: __m256i, b: __m256i) -> __m256i { let sum = _mm256_add_epi16(a, b); let ge_q = _mm256_cmpgt_epi16(sum, _mm256_set1_epi16(Q_I16 - 1)); @@ -369,6 +487,9 @@ fn add_mod_u16x16_avx2(a: __m256i, b: __m256i) -> __m256i { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn sub_mod_u16x16_avx2(a: __m256i, b: __m256i) -> __m256i { let diff = _mm256_sub_epi16(a, b); let borrowed = _mm256_cmpgt_epi16(b, a); @@ -376,6 +497,10 @@ fn sub_mod_u16x16_avx2(a: __m256i, b: __m256i) -> __m256i { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `coeff_offset..coeff_offset + 16` must be in +/// bounds for both polynomials. fn base_multiply_chunk_avx2(a: &Poly, b: &Poly, gamma: __m256i, coeff_offset: usize) -> (__m256i, __m256i) { let mask = _mm256_set1_epi32(0xffff); // SAFETY: fixed-size AVX2 polynomial chunk loads because: @@ -383,8 +508,8 @@ fn base_multiply_chunk_avx2(a: &Poly, b: &Poly, gamma: __m256i, coeff_offset: us // 2. Each load touches `coeff_offset..coeff_offset + 16`, within each 256-coefficient polynomial. // 3. `_mm256_loadu_si256` accepts arbitrary alignment. unsafe { - let a_pairs = _mm256_loadu_si256(a.as_ptr().add(coeff_offset).cast::<__m256i>()); - let b_pairs = _mm256_loadu_si256(b.as_ptr().add(coeff_offset).cast::<__m256i>()); + let a_pairs = load_u16x16(a.as_ptr().add(coeff_offset)); + let b_pairs = load_u16x16(b.as_ptr().add(coeff_offset)); let a0 = _mm256_and_si256(a_pairs, mask); let a1 = _mm256_srli_epi32::<16>(a_pairs); let b0 = _mm256_and_si256(b_pairs, mask); @@ -403,36 +528,52 @@ fn base_multiply_chunk_avx2(a: &Poly, b: &Poly, gamma: __m256i, coeff_offset: us } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn acc_lanes0_avx2(acc_pairs: __m256i) -> __m256i { _mm256_and_si256(acc_pairs, _mm256_set1_epi32(0xffff)) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn acc_lanes1_avx2(acc_pairs: __m256i) -> __m256i { _mm256_srli_epi32::<16>(acc_pairs) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to write 16 `u16` values. fn store_u32_pair_lanes_as_u16_avx2(ptr: *mut u16, lo: __m256i, hi: __m256i) { let packed = _mm256_or_si256(lo, _mm256_slli_epi32::<16>(hi)); // SAFETY: fixed-size AVX2 polynomial chunk store because: // 1. The caller passes a pointer to the start of a 16-coefficient in-bounds chunk. // 2. `lo` and `hi` are reduced modulo Q, so each lane fits in u16 before packing. // 3. `_mm256_storeu_si256` accepts arbitrary alignment. - unsafe { _mm256_storeu_si256(ptr.cast::<__m256i>(), packed) }; + unsafe { store_u16x16(ptr, packed) }; } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to read eight initialized +/// `i16` values. fn load_i16x8_as_i32x8_avx2(ptr: *const i16) -> __m256i { // SAFETY: unaligned 8-coefficient input load because: // 1. The caller proves `ptr..ptr + 8` is readable. // 2. `_mm_loadu_si128` accepts arbitrary alignment. // 3. `_mm256_cvtepi16_epi32` sign-extends the 8 i16 lanes into 8 i32 lanes. - let packed = unsafe { _mm_loadu_si128(ptr.cast::<__m128i>()) }; + let packed = unsafe { load_i16x8(ptr) }; _mm256_cvtepi16_epi32(packed) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn montgomery_reduce_i32x8_avx2(value: __m256i) -> __m256i { let k = _mm256_mullo_epi16(value, _mm256_set1_epi32(i32::from(Q_MONT_INV_U16))); let c = _mm256_mulhi_epi16(k, _mm256_set1_epi32(Q_I32)); @@ -442,19 +583,29 @@ fn montgomery_reduce_i32x8_avx2(value: __m256i) -> __m256i { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn signed_to_mod_q_i32x8_avx2(value: __m256i) -> __m256i { let negative = _mm256_cmpgt_epi32(_mm256_setzero_si256(), value); _mm256_add_epi32(value, _mm256_and_si256(negative, _mm256_set1_epi32(Q_I32))) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn add_mod_u32x8_avx2(a: __m256i, b: __m256i) -> __m256i { let sum = _mm256_add_epi32(a, b); - let ge_q = _mm256_cmpgt_epi32(sum, _mm256_set1_epi32((Q_U32 - 1) as i32)); + let ge_q = _mm256_cmpgt_epi32(sum, _mm256_set1_epi32((Q_U32 - 1).cast_signed())); _mm256_sub_epi32(sum, _mm256_and_si256(ge_q, _mm256_set1_epi32(Q_I32))) } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1, and +/// `coeff_offset..coeff_offset + 32` must be in bounds for both polynomials. fn base_multiply_chunk_avx512(a: &Poly, b: &Poly, gamma: __m512i, coeff_offset: usize) -> (__m512i, __m512i) { let mask = _mm512_set1_epi32(0xffff); // SAFETY: fixed-size AVX-512 polynomial chunk loads because: @@ -462,8 +613,8 @@ fn base_multiply_chunk_avx512(a: &Poly, b: &Poly, gamma: __m512i, coeff_offset: // 2. Each load touches `coeff_offset..coeff_offset + 32`, within each 256-coefficient polynomial. // 3. `_mm512_loadu_si512` accepts arbitrary alignment. unsafe { - let a_pairs = _mm512_loadu_si512(a.as_ptr().add(coeff_offset).cast::<__m512i>()); - let b_pairs = _mm512_loadu_si512(b.as_ptr().add(coeff_offset).cast::<__m512i>()); + let a_pairs = load_u16x32(a.as_ptr().add(coeff_offset)); + let b_pairs = load_u16x32(b.as_ptr().add(coeff_offset)); let a0 = _mm512_and_si512(a_pairs, mask); let a1 = _mm512_srli_epi32::<16>(a_pairs); let b0 = _mm512_and_si512(b_pairs, mask); @@ -482,36 +633,53 @@ fn base_multiply_chunk_avx512(a: &Poly, b: &Poly, gamma: __m512i, coeff_offset: } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. fn acc_lanes0_avx512(acc_pairs: __m512i) -> __m512i { _mm512_and_si512(acc_pairs, _mm512_set1_epi32(0xffff)) } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. fn acc_lanes1_avx512(acc_pairs: __m512i) -> __m512i { _mm512_srli_epi32::<16>(acc_pairs) } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1, and `ptr` must be +/// valid to write 32 `u16` values. fn store_u32_pair_lanes_as_u16_avx512(ptr: *mut u16, lo: __m512i, hi: __m512i) { let packed = _mm512_or_si512(lo, _mm512_slli_epi32::<16>(hi)); // SAFETY: fixed-size AVX-512 polynomial chunk store because: // 1. The caller passes a pointer to the start of a 32-coefficient in-bounds chunk. // 2. `lo` and `hi` are reduced modulo Q, so each lane fits in u16 before packing. // 3. `_mm512_storeu_si512` accepts arbitrary alignment. - unsafe { _mm512_storeu_si512(ptr.cast::<__m512i>(), packed) }; + unsafe { store_u16x32(ptr, packed) }; } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1, and `ptr` must be +/// valid to read 16 initialized `i16` values. fn load_i16x16_as_i32x16_avx512(ptr: *const i16) -> __m512i { // SAFETY: unaligned 16-coefficient input load because: // 1. The caller proves `ptr..ptr + 16` is readable. // 2. `_mm256_loadu_si256` accepts arbitrary alignment. // 3. `_mm512_cvtepi16_epi32` sign-extends the 16 i16 lanes into 16 i32 lanes. - let packed = unsafe { _mm256_loadu_si256(ptr.cast::<__m256i>()) }; + let packed = unsafe { load_i16x16(ptr) }; _mm512_cvtepi16_epi32(packed) } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. fn montgomery_reduce_i32x16_avx512(value: __m512i) -> __m512i { let k = _mm512_mullo_epi16(value, _mm512_set1_epi32(i32::from(Q_MONT_INV_U16))); let c = _mm512_mulhi_epi16(k, _mm512_set1_epi32(Q_I32)); @@ -521,14 +689,20 @@ fn montgomery_reduce_i32x16_avx512(value: __m512i) -> __m512i { } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. fn signed_to_mod_q_i32x16_avx512(value: __m512i) -> __m512i { let negative = _mm512_cmpgt_epi32_mask(_mm512_setzero_si512(), value); _mm512_mask_add_epi32(value, negative, value, _mm512_set1_epi32(Q_I32)) } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. fn add_mod_u32x16_avx512(a: __m512i, b: __m512i) -> __m512i { let sum = _mm512_add_epi32(a, b); - let ge_q = _mm512_cmpgt_epi32_mask(sum, _mm512_set1_epi32((Q_U32 - 1) as i32)); + let ge_q = _mm512_cmpgt_epi32_mask(sum, _mm512_set1_epi32((Q_U32 - 1).cast_signed())); _mm512_mask_sub_epi32(sum, ge_q, sum, _mm512_set1_epi32(Q_I32)) } diff --git a/src/auth/mod.rs b/src/auth/mod.rs index ae8e1925..f50505dc 100644 --- a/src/auth/mod.rs +++ b/src/auth/mod.rs @@ -47,28 +47,28 @@ //! ```toml //! [dependencies] //! # MAC bundle: HMAC-SHA-2/SHA-3, KMAC128/256, and standalone Poly1305 -//! rscrypto = { version = "0.6.4", default-features = false, features = ["macs"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["macs"] } //! //! # HKDF only -//! rscrypto = { version = "0.6.4", default-features = false, features = ["hkdf"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["hkdf"] } //! //! # Ed25519 only -//! rscrypto = { version = "0.6.4", default-features = false, features = ["ed25519"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["ed25519"] } //! //! # ECDSA P-256/P-384 signing and verification -//! rscrypto = { version = "0.6.4", default-features = false, features = ["ecdsa"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["ecdsa"] } //! //! # RSA -//! rscrypto = { version = "0.6.4", default-features = false, features = ["rsa"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["rsa"] } //! //! # Signature primitives -//! rscrypto = { version = "0.6.4", default-features = false, features = ["signatures"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["signatures"] } //! //! # X25519 only -//! rscrypto = { version = "0.6.4", default-features = false, features = ["key-exchange"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["key-exchange"] } //! //! # Everything in auth/key-derivation -//! rscrypto = { version = "0.6.4", default-features = false, features = ["auth"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["auth"] } //! ``` //! //! # API Conventions @@ -115,7 +115,24 @@ #[cfg(feature = "argon2")] pub mod argon2; -#[cfg(any(feature = "ed25519", feature = "x25519"))] +#[cfg(any( + feature = "ed25519", + all( + feature = "x25519", + any( + test, + miri, + not(any( + all( + target_arch = "aarch64", + any(target_os = "macos", target_os = "linux"), + not(feature = "portable-only") + ), + all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) + )) + ) + ) +))] pub(crate) mod curve25519_edwards; #[cfg(any(feature = "ecdsa-p256", feature = "ecdsa-p384"))] pub mod ecdsa; @@ -133,7 +150,7 @@ pub mod kmac; pub mod mlkem; #[cfg(feature = "pbkdf2")] pub mod pbkdf2; -#[cfg(feature = "phc-strings")] +#[cfg(all(feature = "phc-strings", any(feature = "argon2", feature = "scrypt")))] pub(crate) mod phc; #[cfg(feature = "poly1305")] pub mod poly1305; @@ -297,10 +314,10 @@ pub use pbkdf2::{diag_pbkdf2_sha256_verify_portable, diag_pbkdf2_sha512_verify_p pub use poly1305::{Poly1305, Poly1305OneTimeKey, Poly1305Tag}; #[cfg(feature = "rsa")] pub use rsa::{ - RsaEncryptionError, RsaJwtAlgorithm, RsaJwtVerifier, RsaKeyError, RsaKeyGenerationContract, RsaKeyGenerationError, - RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPrivateOpError, RsaPrivateScratch, - RsaProtocolAlgorithmError, RsaPssProfile, RsaPublicExponent, RsaPublicExponentPolicy, RsaPublicKey, - RsaPublicKeyPolicy, RsaPublicOpError, RsaPublicScratch, RsaSignatureProfile, RsaSignatureSigner, + RsaBlindingPair, RsaEncryptionError, RsaJwtAlgorithm, RsaJwtVerifier, RsaKeyError, RsaKeyGenerationContract, + RsaKeyGenerationError, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPrivateOpError, + RsaPrivateScratch, RsaProtocolAlgorithmError, RsaPssProfile, RsaPublicExponent, RsaPublicExponentPolicy, + RsaPublicKey, RsaPublicKeyPolicy, RsaPublicOpError, RsaPublicScratch, RsaSignatureProfile, RsaSignatureSigner, RsaSignatureVerifier, RsaTlsSignatureSchemes, RsaX509PublicKey, RsaX509PublicKeyAlgorithm, }; #[cfg(all(feature = "rsa", feature = "diag"))] diff --git a/src/auth/pbkdf2.rs b/src/auth/pbkdf2.rs index c55bab03..d84c627c 100644 --- a/src/auth/pbkdf2.rs +++ b/src/auth/pbkdf2.rs @@ -60,7 +60,6 @@ const SHA512_BLOCK_SIZE: usize = 128; const SHA512_INLINE_SALT_MAX: usize = SHA512_BLOCK_SIZE - 4 - 1 - 16; #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_u32x8_be(dst: &mut [u8], words: &[u32; 8]) { dst[0..4].copy_from_slice(&words[0].to_be_bytes()); dst[4..8].copy_from_slice(&words[1].to_be_bytes()); @@ -73,7 +72,6 @@ fn write_u32x8_be(dst: &mut [u8], words: &[u32; 8]) { } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_u64x8_be(dst: &mut [u8], words: &[u64; 8]) { dst[0..8].copy_from_slice(&words[0].to_be_bytes()); dst[8..16].copy_from_slice(&words[1].to_be_bytes()); @@ -339,7 +337,6 @@ macro_rules! define_pbkdf2_sha2 { /// Pre-compute HMAC prefix states from `password`. #[must_use] - #[allow(clippy::indexing_slicing)] // password.len() <= block size in the else branch. pub fn new(password: &[u8]) -> Self { if let Some(state) = $new_fast_path(password) { return state; @@ -375,20 +372,17 @@ macro_rules! define_pbkdf2_sha2 { /// Derive a key into `okm`. #[inline] - #[allow(clippy::indexing_slicing)] pub fn derive(&self, salt: &[u8], iterations: u32, okm: &mut [u8]) -> Result<(), Pbkdf2Error> { Self::derive_with_prefixes(self.compress, &self.inner_init, &self.outer_init, salt, iterations, okm) } /// Derive a key into `okm` using validated password parameters. #[inline] - #[allow(clippy::indexing_slicing)] pub fn derive_with_params(&self, params: Pbkdf2Params<'_>, okm: &mut [u8]) -> Result<(), Pbkdf2Error> { self.derive(params.salt(), params.iterations(), okm) } #[inline] - #[allow(clippy::indexing_slicing)] fn derive_with_prefixes( compress: $compress_ty, inner_init: &[$word_ty; 8], @@ -403,23 +397,17 @@ macro_rules! define_pbkdf2_sha2 { if okm.is_empty() { return Ok(()); } - let num_blocks = okm.len().div_ceil($output_size_const); - if num_blocks as u64 > u32::MAX as u64 { - return Err(Pbkdf2Error::OutputTooLong); - } + let num_blocks = u32::try_from(okm.len().div_ceil($output_size_const)) + .map_err(|_| Pbkdf2Error::OutputTooLong)?; if iterations == 1 { - $iter1_fn(compress, inner_init, outer_init, salt, okm); + $iter1_fn(compress, inner_init, outer_init, salt, num_blocks, okm); return Ok(()); } - let mut block_index = 1u32; - let mut chunks = okm.chunks_exact_mut($output_size_const); + let (chunks, tail) = okm.as_chunks_mut::<$output_size_const>(); - for chunk in chunks.by_ref() { - // SAFETY: chunks_exact_mut yields slices whose length is exactly - // $output_size_const, so this cast is to the same initialized bytes. - let full_chunk = unsafe { &mut *(chunk.as_mut_ptr().cast::<[u8; $output_size_const]>()) }; + for (block_index, chunk) in (1..=num_blocks).zip(chunks) { $f_fn( compress, inner_init, @@ -427,12 +415,10 @@ macro_rules! define_pbkdf2_sha2 { salt, iterations, block_index, - full_chunk, + chunk, ); - block_index = block_index.strict_add(1); } - let tail = chunks.into_remainder(); if !tail.is_empty() { let mut block_out = [0u8; $output_size_const]; $f_fn( @@ -441,7 +427,7 @@ macro_rules! define_pbkdf2_sha2 { outer_init, salt, iterations, - block_index, + num_blocks, &mut block_out, ); tail.copy_from_slice(&block_out[..tail.len()]); @@ -470,7 +456,6 @@ macro_rules! define_pbkdf2_sha2 { /// /// Generated-code timing claims are configuration- and release-evidence-bound; /// see `ct.toml`. - #[allow(clippy::indexing_slicing)] #[must_use = "password verification must be checked; a dropped Result silently accepts the wrong password"] pub fn verify(&self, salt: &[u8], iterations: u32, expected: &[u8]) -> Result<(), VerificationError> { self.verify_with_policy_bounded( @@ -491,7 +476,6 @@ macro_rules! define_pbkdf2_sha2 { /// /// Generated-code timing claims are configuration- and release-evidence-bound; /// see `ct.toml`. - #[allow(clippy::indexing_slicing)] #[must_use = "password verification must be checked; a dropped Result silently accepts the wrong password"] pub fn verify_with_policy( &self, @@ -506,7 +490,6 @@ macro_rules! define_pbkdf2_sha2 { /// Verify `expected` under an explicit lower-bound policy and /// caller-selected verification work limit. - #[allow(clippy::indexing_slicing)] #[must_use = "password verification must be checked; a dropped Result silently accepts the wrong password"] pub fn verify_with_policy_bounded( &self, @@ -536,24 +519,20 @@ macro_rules! define_pbkdf2_sha2 { /// verification should use [`verify`](Self::verify), /// [`verify_with_policy`](Self::verify_with_policy), or /// [`verify_password`](Self::verify_password). - #[allow(clippy::indexing_slicing)] #[must_use = "password verification must be checked; a dropped Result silently accepts the wrong password"] pub fn verify_primitive(&self, salt: &[u8], iterations: u32, expected: &[u8]) -> Result<(), VerificationError> { if iterations == 0 || expected.is_empty() { return Err(VerificationError::new()); } - let num_blocks = expected.len().div_ceil($output_size_const); - if num_blocks as u64 > u32::MAX as u64 { - return Err(VerificationError::new()); - } + let num_blocks = u32::try_from(expected.len().div_ceil($output_size_const)) + .map_err(|_| VerificationError::new())?; let compress = self.compress; let mut block_out = [0u8; $output_size_const]; let mut acc = 0u8; - for (i, chunk) in expected.chunks($output_size_const).enumerate() { - let block_index = (i as u32).strict_add(1); + for (block_index, chunk) in (1..=num_blocks).zip(expected.chunks($output_size_const)) { $f_fn( compress, &self.inner_init, @@ -716,8 +695,6 @@ macro_rules! define_pbkdf2_sha2 { /// Test-only: build with a specific digest compress function. #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] - #[allow(clippy::indexing_slicing)] pub(crate) fn new_with_compress_for_test(password: &[u8], compress: $compress_ty) -> Self { let mut key_block = [0u8; $block_size_const]; if password.len() > $block_size_const { @@ -804,7 +781,6 @@ define_pbkdf2_sha2! { /// Test-only: one-shot SHA-256 digest using a specific compress function. #[cfg(any(test, feature = "diag"))] -#[allow(clippy::indexing_slicing)] fn sha256_oneshot_with_compress(data: &[u8], compress: Sha256CompressBlocksFn) -> [u8; SHA256_OUTPUT_SIZE] { let mut state = SHA256_H0; let mut pos = 0usize; @@ -823,7 +799,8 @@ fn sha256_oneshot_with_compress(data: &[u8], compress: Sha256CompressBlocksFn) - block[56..64].copy_from_slice(&(data.len() as u64).strict_mul(8).to_be_bytes()); compress(&mut state, &block); let mut out = [0u8; SHA256_OUTPUT_SIZE]; - for (chunk, &word) in out.chunks_exact_mut(4).zip(state.iter()) { + let (chunks, _) = out.as_chunks_mut::<4>(); + for (chunk, &word) in chunks.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } ct::zeroize_words_no_fence(&mut state); @@ -834,6 +811,8 @@ fn sha256_oneshot_with_compress(data: &[u8], compress: Sha256CompressBlocksFn) - #[cfg(feature = "diag")] #[must_use] +/// Return whether portable PBKDF2-HMAC-SHA256 derives `expected` from `password` +/// using the salt `salt` and one iteration. pub fn diag_pbkdf2_sha256_verify_portable( password: &[u8; SHA256_OUTPUT_SIZE], expected: &[u8; SHA256_OUTPUT_SIZE], @@ -848,6 +827,8 @@ pub fn diag_pbkdf2_sha256_verify_portable( #[cfg(feature = "diag")] #[must_use] +/// Return whether portable PBKDF2-HMAC-SHA512 derives `expected` from `password` +/// using the salt `salt` and one iteration. pub fn diag_pbkdf2_sha512_verify_portable( password: &[u8; SHA512_OUTPUT_SIZE], expected: &[u8; SHA512_OUTPUT_SIZE], @@ -861,7 +842,6 @@ pub fn diag_pbkdf2_sha512_verify_portable( } #[inline] -#[allow(clippy::indexing_slicing)] fn pbkdf2_sha256_new_fast_path(password: &[u8]) -> Option { if !pbkdf2_sha256_spr_prefers_hmac_iter1() { return None; @@ -913,7 +893,6 @@ fn pbkdf2_sha256_spr_prefers_hmac_iter1() -> bool { } } -#[allow(clippy::indexing_slicing)] #[inline] fn pbkdf2_sha256_hmac_iter1_small(password: &[u8], salt: &[u8], okm: &mut [u8]) { debug_assert!(salt.len() <= SHA256_INLINE_SALT_MAX); @@ -923,8 +902,7 @@ fn pbkdf2_sha256_hmac_iter1_small(password: &[u8], salt: &[u8], okm: &mut [u8]) msg[..salt.len()].copy_from_slice(salt); let msg_len = salt.len().strict_add(4); - for (i, chunk) in okm.chunks_mut(SHA256_OUTPUT_SIZE).enumerate() { - let block_index = (i as u32).strict_add(1); + for (block_index, chunk) in (1u32..=2).zip(okm.chunks_mut(SHA256_OUTPUT_SIZE)) { msg[salt.len()..msg_len].copy_from_slice(&block_index.to_be_bytes()); let tag = HmacSha256::mac(password, &msg[..msg_len]); chunk.copy_from_slice(&tag.as_bytes()[..chunk.len()]); @@ -974,7 +952,6 @@ fn pbkdf2_sha256_derive_key_fast_path_with_preference( /// Each HMAC iteration in the hot loop runs exactly 2 SHA-256 compress calls /// using pre-padded block templates — no hash struct creation, no dispatch /// overhead, no padding recomputation. -#[allow(clippy::indexing_slicing)] #[inline(always)] fn pbkdf2_sha256_f( compress: Sha256CompressBlocksFn, @@ -1104,28 +1081,23 @@ fn pbkdf2_sha256_f( core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } -#[allow(clippy::indexing_slicing)] #[inline(always)] fn pbkdf2_sha256_iter1( compress: Sha256CompressBlocksFn, inner_init: &[u32; 8], outer_init: &[u32; 8], salt: &[u8], + num_blocks: u32, okm: &mut [u8], ) { if salt.len() > SHA256_INLINE_SALT_MAX { - let mut block_index = 1u32; - let mut chunks = okm.chunks_exact_mut(SHA256_OUTPUT_SIZE); - for chunk in chunks.by_ref() { - // SAFETY: chunks_exact_mut yields slices whose length is exactly SHA256_OUTPUT_SIZE. - let full_chunk = unsafe { &mut *(chunk.as_mut_ptr().cast::<[u8; SHA256_OUTPUT_SIZE]>()) }; - pbkdf2_sha256_f(compress, inner_init, outer_init, salt, 1, block_index, full_chunk); - block_index = block_index.strict_add(1); + let (chunks, tail) = okm.as_chunks_mut::(); + for (block_index, chunk) in (1..=num_blocks).zip(chunks) { + pbkdf2_sha256_f(compress, inner_init, outer_init, salt, 1, block_index, chunk); } - let tail = chunks.into_remainder(); if !tail.is_empty() { let mut block_out = [0u8; SHA256_OUTPUT_SIZE]; - pbkdf2_sha256_f(compress, inner_init, outer_init, salt, 1, block_index, &mut block_out); + pbkdf2_sha256_f(compress, inner_init, outer_init, salt, 1, num_blocks, &mut block_out); tail.copy_from_slice(&block_out[..tail.len()]); ct::zeroize(&mut block_out); } @@ -1147,10 +1119,9 @@ fn pbkdf2_sha256_iter1( outer_block[56..SHA256_BLOCK_SIZE].copy_from_slice(&768u64.to_be_bytes()); let mut state = [0u32; 8]; - let mut block_index = 1u32; - let mut chunks = okm.chunks_exact_mut(SHA256_OUTPUT_SIZE); - for chunk in chunks.by_ref() { + let (chunks, tail) = okm.as_chunks_mut::(); + for (block_index, chunk) in (1..=num_blocks).zip(chunks) { block[index_pos..pad_pos].copy_from_slice(&block_index.to_be_bytes()); state = *inner_init; @@ -1161,12 +1132,10 @@ fn pbkdf2_sha256_iter1( compress(&mut state, &outer_block); write_u32x8_be(chunk, &state); - block_index = block_index.strict_add(1); } - let tail = chunks.into_remainder(); if !tail.is_empty() { - block[index_pos..pad_pos].copy_from_slice(&block_index.to_be_bytes()); + block[index_pos..pad_pos].copy_from_slice(&num_blocks.to_be_bytes()); state = *inner_init; compress(&mut state, &block); @@ -1239,8 +1208,6 @@ fn pbkdf2_sha512_derive_key_fast_path( /// Test-only: one-shot SHA-512 digest using a specific compress function. #[cfg(any(test, feature = "diag"))] -#[allow(dead_code)] -#[allow(clippy::indexing_slicing)] fn sha512_oneshot_with_compress(data: &[u8], compress: Sha512CompressBlocksFn) -> [u8; SHA512_OUTPUT_SIZE] { let mut state = SHA512_H0; let mut pos = 0usize; @@ -1259,7 +1226,8 @@ fn sha512_oneshot_with_compress(data: &[u8], compress: Sha512CompressBlocksFn) - block[112..128].copy_from_slice(&(data.len() as u128).strict_mul(8).to_be_bytes()); compress(&mut state, &block); let mut out = [0u8; SHA512_OUTPUT_SIZE]; - for (chunk, &word) in out.chunks_exact_mut(8).zip(state.iter()) { + let (chunks, _) = out.as_chunks_mut::<8>(); + for (chunk, &word) in chunks.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } ct::zeroize_words_no_fence(&mut state); @@ -1269,7 +1237,6 @@ fn sha512_oneshot_with_compress(data: &[u8], compress: Sha512CompressBlocksFn) - } /// Compute one PBKDF2-SHA512 block: `F(Password, Salt, c, i)`. -#[allow(clippy::indexing_slicing)] #[inline(always)] fn pbkdf2_sha512_f( compress: Sha512CompressBlocksFn, @@ -1382,28 +1349,23 @@ fn pbkdf2_sha512_f( core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } -#[allow(clippy::indexing_slicing)] #[inline(always)] fn pbkdf2_sha512_iter1( compress: Sha512CompressBlocksFn, inner_init: &[u64; 8], outer_init: &[u64; 8], salt: &[u8], + num_blocks: u32, okm: &mut [u8], ) { if salt.len() > SHA512_INLINE_SALT_MAX { - let mut block_index = 1u32; - let mut chunks = okm.chunks_exact_mut(SHA512_OUTPUT_SIZE); - for chunk in chunks.by_ref() { - // SAFETY: chunks_exact_mut yields slices whose length is exactly SHA512_OUTPUT_SIZE. - let full_chunk = unsafe { &mut *(chunk.as_mut_ptr().cast::<[u8; SHA512_OUTPUT_SIZE]>()) }; - pbkdf2_sha512_f(compress, inner_init, outer_init, salt, 1, block_index, full_chunk); - block_index = block_index.strict_add(1); + let (chunks, tail) = okm.as_chunks_mut::(); + for (block_index, chunk) in (1..=num_blocks).zip(chunks) { + pbkdf2_sha512_f(compress, inner_init, outer_init, salt, 1, block_index, chunk); } - let tail = chunks.into_remainder(); if !tail.is_empty() { let mut block_out = [0u8; SHA512_OUTPUT_SIZE]; - pbkdf2_sha512_f(compress, inner_init, outer_init, salt, 1, block_index, &mut block_out); + pbkdf2_sha512_f(compress, inner_init, outer_init, salt, 1, num_blocks, &mut block_out); tail.copy_from_slice(&block_out[..tail.len()]); ct::zeroize(&mut block_out); } @@ -1425,10 +1387,9 @@ fn pbkdf2_sha512_iter1( outer_block[112..SHA512_BLOCK_SIZE].copy_from_slice(&1536u128.to_be_bytes()); let mut state = [0u64; 8]; - let mut block_index = 1u32; - let mut chunks = okm.chunks_exact_mut(SHA512_OUTPUT_SIZE); - for chunk in chunks.by_ref() { + let (chunks, tail) = okm.as_chunks_mut::(); + for (block_index, chunk) in (1..=num_blocks).zip(chunks) { block[index_pos..pad_pos].copy_from_slice(&block_index.to_be_bytes()); state = *inner_init; @@ -1439,12 +1400,10 @@ fn pbkdf2_sha512_iter1( compress(&mut state, &outer_block); write_u64x8_be(chunk, &state); - block_index = block_index.strict_add(1); } - let tail = chunks.into_remainder(); if !tail.is_empty() { - block[index_pos..pad_pos].copy_from_slice(&block_index.to_be_bytes()); + block[index_pos..pad_pos].copy_from_slice(&num_blocks.to_be_bytes()); state = *inner_init; compress(&mut state, &block); @@ -1477,7 +1436,7 @@ mod tests { #[test] fn rfc7914_sha256_vector_1() { let mut dk = [0u8; 64]; - Pbkdf2Sha256::derive_key_primitive(b"passwd", b"salt", 1, &mut dk).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"passwd", b"salt", 1, &mut dk).expect("RFC 7914 vector parameters are valid"); assert_eq!( dk, [ @@ -1498,7 +1457,9 @@ mod tests { for len in [1usize, 16, 31, 32, 33, 63, 64] { let mut expected = vec![0u8; len]; let mut actual = vec![0u8; len]; - state.derive(&salt, 1, &mut expected).unwrap(); + state + .derive(&salt, 1, &mut expected) + .expect("nonempty output and one iteration are valid"); pbkdf2_sha256_hmac_iter1_small(&password, &salt, &mut actual); assert_eq!(actual, expected, "len={len}"); } @@ -1512,20 +1473,33 @@ mod tests { let mut actual = [0u8; 64]; oracle_sha256(&password, &salt, 1, &mut expected); - assert!(pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 1, &mut actual, true).unwrap()); + assert!( + pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 1, &mut actual, true) + .expect("fast-path parameters are valid") + ); assert_eq!(actual, expected); let mut rejected = [0u8; 65]; - assert!(!pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 1, &mut rejected, true).unwrap()); - assert!(!pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 2, &mut actual, true).unwrap()); - assert!(!pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 1, &mut actual, false).unwrap()); + assert!( + !pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 1, &mut rejected, true) + .expect("oversized fast-path output is a supported fallback") + ); + assert!( + !pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 2, &mut actual, true) + .expect("multi-iteration fast-path input is a supported fallback") + ); + assert!( + !pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 1, &mut actual, false) + .expect("disabled fast-path input is a supported fallback") + ); } #[cfg(not(miri))] #[test] fn rfc7914_sha256_vector_2() { let mut dk = [0u8; 64]; - Pbkdf2Sha256::derive_key_primitive(b"Password", b"NaCl", 80000, &mut dk).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"Password", b"NaCl", 80000, &mut dk) + .expect("RFC 7914 vector parameters are valid"); assert_eq!( dk, [ @@ -1589,7 +1563,8 @@ mod tests { oracle_sha256(password, salt, iterations, &mut expected); let mut actual = vec![0u8; dk_len]; - Pbkdf2Sha256::derive_key_primitive(password, salt, iterations, &mut actual).unwrap(); + Pbkdf2Sha256::derive_key_primitive(password, salt, iterations, &mut actual) + .expect("oracle case parameters are valid"); assert_eq!( actual, @@ -1642,7 +1617,8 @@ mod tests { oracle_sha512(password, salt, iterations, &mut expected); let mut actual = vec![0u8; dk_len]; - Pbkdf2Sha512::derive_key_primitive(password, salt, iterations, &mut actual).unwrap(); + Pbkdf2Sha512::derive_key_primitive(password, salt, iterations, &mut actual) + .expect("oracle case parameters are valid"); assert_eq!( actual, @@ -1660,37 +1636,45 @@ mod tests { #[test] fn sha256_verify_correct_password() { - let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100).unwrap(); - assert!(Pbkdf2Sha256::verify_password_primitive(b"password", b"salt", 100, &dk).is_ok()); + let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100) + .expect("test derivation parameters are valid"); + Pbkdf2Sha256::verify_password_primitive(b"password", b"salt", 100, &dk) + .expect("matching PBKDF2-SHA256 output must verify"); } #[test] fn sha256_verify_wrong_password() { - let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100).unwrap(); + let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100) + .expect("test derivation parameters are valid"); assert!(Pbkdf2Sha256::verify_password_primitive(b"wrong", b"salt", 100, &dk).is_err()); } #[test] fn sha256_verify_wrong_salt() { - let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100).unwrap(); + let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100) + .expect("test derivation parameters are valid"); assert!(Pbkdf2Sha256::verify_password_primitive(b"password", b"wrong", 100, &dk).is_err()); } #[test] fn sha256_verify_wrong_iterations() { - let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100).unwrap(); + let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100) + .expect("test derivation parameters are valid"); assert!(Pbkdf2Sha256::verify_password_primitive(b"password", b"salt", 101, &dk).is_err()); } #[test] fn sha512_verify_correct_password() { - let dk = Pbkdf2Sha512::derive_key_array_primitive::<64>(b"password", b"salt", 100).unwrap(); - assert!(Pbkdf2Sha512::verify_password_primitive(b"password", b"salt", 100, &dk).is_ok()); + let dk = Pbkdf2Sha512::derive_key_array_primitive::<64>(b"password", b"salt", 100) + .expect("test derivation parameters are valid"); + Pbkdf2Sha512::verify_password_primitive(b"password", b"salt", 100, &dk) + .expect("matching PBKDF2-SHA512 output must verify"); } #[test] fn sha512_verify_wrong_password() { - let dk = Pbkdf2Sha512::derive_key_array_primitive::<64>(b"password", b"salt", 100).unwrap(); + let dk = Pbkdf2Sha512::derive_key_array_primitive::<64>(b"password", b"salt", 100) + .expect("test derivation parameters are valid"); assert!(Pbkdf2Sha512::verify_password_primitive(b"wrong", b"salt", 100, &dk).is_err()); } @@ -1728,15 +1712,13 @@ mod tests { ), Err(Pbkdf2Error::SaltTooShort) ); - assert!( - Pbkdf2Sha256::derive_key( - b"pw", - &strong_salt, - Pbkdf2Sha256::MIN_RECOMMENDED_ITERATIONS, - &mut empty - ) - .is_ok() - ); + Pbkdf2Sha256::derive_key( + b"pw", + &strong_salt, + Pbkdf2Sha256::MIN_RECOMMENDED_ITERATIONS, + &mut empty, + ) + .expect("the default PBKDF2-SHA256 policy must accept its minimums"); assert!( Pbkdf2Sha256::verify_password( b"pw", @@ -1751,12 +1733,14 @@ mod tests { #[test] fn pbkdf2_params_can_use_explicit_policy_for_migrations() { let policy = Pbkdf2VerifyPolicy::new(1, 0); - let params = Pbkdf2Sha256::params_with_policy(b"", 1, &policy).unwrap(); + let params = + Pbkdf2Sha256::params_with_policy(b"", 1, &policy).expect("the explicit migration policy permits these inputs"); assert_eq!(params.salt(), b""); assert_eq!(params.iterations(), 1); let mut empty = []; - assert!(Pbkdf2Sha256::derive_key_with_params(b"pw", params, &mut empty).is_ok()); + Pbkdf2Sha256::derive_key_with_params(b"pw", params, &mut empty) + .expect("validated migration parameters permit empty output"); } #[test] @@ -1769,11 +1753,12 @@ mod tests { ] { let params = Pbkdf2Sha256::params_with_policy_bounded(b"", iterations, &policy, Pbkdf2Sha256::MAX_VERIFY_ITERATIONS) - .unwrap(); + .expect("iterations at or below the verification limit are valid"); assert!(policy.allows_bounded(¶ms, Pbkdf2Sha256::MAX_VERIFY_ITERATIONS)); } let excessive_sha256 = Pbkdf2Sha256::MAX_VERIFY_ITERATIONS.strict_add(1); - let lower_bound_only = Pbkdf2Sha256::params_with_policy(b"", excessive_sha256, &policy).unwrap(); + let lower_bound_only = Pbkdf2Sha256::params_with_policy(b"", excessive_sha256, &policy) + .expect("the lower-bound-only policy has no upper iteration limit"); assert!(policy.allows(&lower_bound_only)); assert!(!policy.allows_bounded(&lower_bound_only, Pbkdf2Sha256::MAX_VERIFY_ITERATIONS)); assert_eq!( @@ -1802,11 +1787,12 @@ mod tests { ] { let params = Pbkdf2Sha512::params_with_policy_bounded(b"", iterations, &policy, Pbkdf2Sha512::MAX_VERIFY_ITERATIONS) - .unwrap(); + .expect("iterations at or below the verification limit are valid"); assert!(policy.allows_bounded(¶ms, Pbkdf2Sha512::MAX_VERIFY_ITERATIONS)); } let excessive_sha512 = Pbkdf2Sha512::MAX_VERIFY_ITERATIONS.strict_add(1); - let lower_bound_only = Pbkdf2Sha512::params_with_policy(b"", excessive_sha512, &policy).unwrap(); + let lower_bound_only = Pbkdf2Sha512::params_with_policy(b"", excessive_sha512, &policy) + .expect("the lower-bound-only policy has no upper iteration limit"); assert!(policy.allows(&lower_bound_only)); assert!(!policy.allows_bounded(&lower_bound_only, Pbkdf2Sha512::MAX_VERIFY_ITERATIONS)); assert_eq!( @@ -1841,12 +1827,12 @@ mod tests { #[test] fn sha256_empty_output_ok() { - assert!(Pbkdf2Sha256::derive_key_primitive(b"pw", b"salt", 1, &mut []).is_ok()); + Pbkdf2Sha256::derive_key_primitive(b"pw", b"salt", 1, &mut []).expect("PBKDF2-SHA256 permits empty output"); } #[test] fn sha512_empty_output_ok() { - assert!(Pbkdf2Sha512::derive_key_primitive(b"pw", b"salt", 1, &mut []).is_ok()); + Pbkdf2Sha512::derive_key_primitive(b"pw", b"salt", 1, &mut []).expect("PBKDF2-SHA512 permits empty output"); } #[test] @@ -1870,8 +1856,10 @@ mod tests { for out_len in output_lengths { let mut expected = vec![0u8; out_len]; - Pbkdf2Sha256::derive_key_primitive(&password, &salt, 2, &mut expected).unwrap(); - assert!(Pbkdf2Sha256::verify_password_primitive(&password, &salt, 2, &expected).is_ok()); + Pbkdf2Sha256::derive_key_primitive(&password, &salt, 2, &mut expected) + .expect("coverage case parameters are valid"); + Pbkdf2Sha256::verify_password_primitive(&password, &salt, 2, &expected) + .expect("matching PBKDF2-SHA256 output must verify"); let mut wrong_first = expected.clone(); wrong_first[0] ^= 1; @@ -1895,8 +1883,10 @@ mod tests { for out_len in output_lengths { let mut expected = vec![0u8; out_len]; - Pbkdf2Sha512::derive_key_primitive(&password, &salt, 2, &mut expected).unwrap(); - assert!(Pbkdf2Sha512::verify_password_primitive(&password, &salt, 2, &expected).is_ok()); + Pbkdf2Sha512::derive_key_primitive(&password, &salt, 2, &mut expected) + .expect("coverage case parameters are valid"); + Pbkdf2Sha512::verify_password_primitive(&password, &salt, 2, &expected) + .expect("matching PBKDF2-SHA512 output must verify"); let mut wrong_first = expected.clone(); wrong_first[0] ^= 1; @@ -1914,11 +1904,17 @@ mod tests { #[test] fn sha256_state_reuse_matches_oneshot() { let state = Pbkdf2Sha256::new(b"password"); - let dk1 = state.derive_array::<32>(b"salt1", 100).unwrap(); - let dk2 = state.derive_array::<32>(b"salt2", 100).unwrap(); - - let oneshot1 = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt1", 100).unwrap(); - let oneshot2 = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt2", 100).unwrap(); + let dk1 = state + .derive_array::<32>(b"salt1", 100) + .expect("first state-reuse derivation parameters are valid"); + let dk2 = state + .derive_array::<32>(b"salt2", 100) + .expect("second state-reuse derivation parameters are valid"); + + let oneshot1 = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt1", 100) + .expect("first one-shot derivation parameters are valid"); + let oneshot2 = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt2", 100) + .expect("second one-shot derivation parameters are valid"); assert_eq!(dk1, oneshot1); assert_eq!(dk2, oneshot2); @@ -1931,7 +1927,8 @@ mod tests { fn sha256_single_iteration() { let mut expected = [0u8; 32]; oracle_sha256(b"pw", b"salt", 1, &mut expected); - let actual = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"pw", b"salt", 1).unwrap(); + let actual = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"pw", b"salt", 1) + .expect("single-iteration PBKDF2-SHA256 parameters are valid"); assert_eq!(actual, expected); } @@ -1939,7 +1936,8 @@ mod tests { fn sha512_single_iteration() { let mut expected = [0u8; 64]; oracle_sha512(b"pw", b"salt", 1, &mut expected); - let actual = Pbkdf2Sha512::derive_key_array_primitive::<64>(b"pw", b"salt", 1).unwrap(); + let actual = Pbkdf2Sha512::derive_key_array_primitive::<64>(b"pw", b"salt", 1) + .expect("single-iteration PBKDF2-SHA512 parameters are valid"); assert_eq!(actual, expected); } @@ -2054,7 +2052,9 @@ mod tests { let state = Pbkdf2Sha256::new_with_compress_for_test(password, compress); let mut actual = vec![0u8; dk_len]; - state.derive(salt, iterations, &mut actual).unwrap(); + state + .derive(salt, iterations, &mut actual) + .expect("forced SHA-256 kernel case parameters are valid"); assert_eq!( actual, @@ -2089,7 +2089,9 @@ mod tests { let state = Pbkdf2Sha512::new_with_compress_for_test(password, compress); let mut actual = vec![0u8; dk_len]; - state.derive(salt, iterations, &mut actual).unwrap(); + state + .derive(salt, iterations, &mut actual) + .expect("forced SHA-512 kernel case parameters are valid"); assert_eq!( actual, @@ -2136,10 +2138,12 @@ mod tests { for out_len in output_lengths { let mut expected = vec![0u8; out_len]; - state.derive(&salt, 3, &mut expected).unwrap(); + state + .derive(&salt, 3, &mut expected) + .expect("SHA-256 work-count case parameters are valid"); let (ok, ok_blocks) = counted_sha256_verify(&state, &salt, 3, &expected); - assert!(ok.is_ok(), "sha256 verify must accept correct output_len={out_len}"); + ok.expect("SHA-256 verification must accept matching output"); let mut wrong_first = expected.clone(); wrong_first[0] ^= 1; @@ -2182,10 +2186,12 @@ mod tests { for out_len in output_lengths { let mut expected = vec![0u8; out_len]; - state.derive(&salt, 3, &mut expected).unwrap(); + state + .derive(&salt, 3, &mut expected) + .expect("SHA-512 work-count case parameters are valid"); let (ok, ok_blocks) = counted_sha512_verify(&state, &salt, 3, &expected); - assert!(ok.is_ok(), "sha512 verify must accept correct output_len={out_len}"); + ok.expect("SHA-512 verification must accept matching output"); let mut wrong_first = expected.clone(); wrong_first[0] ^= 1; diff --git a/src/auth/phc.rs b/src/auth/phc.rs index 73d8b83f..c6115b93 100644 --- a/src/auth/phc.rs +++ b/src/auth/phc.rs @@ -11,16 +11,6 @@ //! //! [phc]: https://github.com/P-H-C/phc-string-format/blob/master/phc-sf-spec.md -#![allow(clippy::indexing_slicing)] -// Base64 and decimal helpers are only reachable when a -// PHC-aware hasher (argon2 or scrypt) is enabled. Without either, the -// helpers are dead code — silence the warning rather than cfg-gate every -// symbol individually. -#![cfg_attr( - all(feature = "phc-strings", not(any(feature = "argon2", feature = "scrypt"))), - allow(dead_code) -)] - use alloc::string::String; use core::fmt; @@ -45,39 +35,27 @@ const B64_DECODE_TABLE: [u8; 256] = { /// Appends to `out` — callers managing multi-segment PHC strings reuse the /// same `String` buffer without intermediate allocation. pub(crate) fn base64_encode_into(bytes: &[u8], out: &mut String) { - let full_triples = bytes.len() / 3; - let tail = bytes.len() % 3; - - for i in 0..full_triples { - let off = i.strict_mul(3); - // SAFETY: off + 3 <= bytes.len() by construction of full_triples. - let b0 = bytes[off] as u32; - let b1 = bytes[off.strict_add(1)] as u32; - let b2 = bytes[off.strict_add(2)] as u32; - let word = (b0 << 16) | (b1 << 8) | b2; - - out.push(B64_ENCODE_TABLE[((word >> 18) & 0x3F) as usize] as char); - out.push(B64_ENCODE_TABLE[((word >> 12) & 0x3F) as usize] as char); - out.push(B64_ENCODE_TABLE[((word >> 6) & 0x3F) as usize] as char); - out.push(B64_ENCODE_TABLE[(word & 0x3F) as usize] as char); - } - - let off = full_triples.strict_mul(3); - match tail { - 1 => { - let b0 = bytes[off] as u32; - out.push(B64_ENCODE_TABLE[((b0 >> 2) & 0x3F) as usize] as char); - out.push(B64_ENCODE_TABLE[((b0 << 4) & 0x3F) as usize] as char); - } - 2 => { - let b0 = bytes[off] as u32; - let b1 = bytes[off.strict_add(1)] as u32; - let word = (b0 << 8) | b1; - out.push(B64_ENCODE_TABLE[((word >> 10) & 0x3F) as usize] as char); - out.push(B64_ENCODE_TABLE[((word >> 4) & 0x3F) as usize] as char); - out.push(B64_ENCODE_TABLE[((word << 2) & 0x3F) as usize] as char); - } - _ => {} + let (triples, tail) = bytes.as_chunks::<3>(); + for &[b0, b1, b2] in triples { + out.push(char::from(B64_ENCODE_TABLE[usize::from(b0 >> 2)])); + out.push(char::from( + B64_ENCODE_TABLE[usize::from(((b0 & 0x03) << 4) | (b1 >> 4))], + )); + out.push(char::from( + B64_ENCODE_TABLE[usize::from(((b1 & 0x0F) << 2) | (b2 >> 6))], + )); + out.push(char::from(B64_ENCODE_TABLE[usize::from(b2 & 0x3F)])); + } + + if let &[b0, b1] = tail { + out.push(char::from(B64_ENCODE_TABLE[usize::from(b0 >> 2)])); + out.push(char::from( + B64_ENCODE_TABLE[usize::from(((b0 & 0x03) << 4) | (b1 >> 4))], + )); + out.push(char::from(B64_ENCODE_TABLE[usize::from((b1 & 0x0F) << 2)])); + } else if let &[b0] = tail { + out.push(char::from(B64_ENCODE_TABLE[usize::from(b0 >> 2)])); + out.push(char::from(B64_ENCODE_TABLE[usize::from((b0 & 0x03) << 4)])); } } @@ -106,41 +84,30 @@ pub(crate) const fn base64_decoded_len(encoded_len: usize) -> usize { /// round-trip). pub(crate) fn base64_decode_into(s: &str, out: &mut [u8]) -> Result { let bytes = s.as_bytes(); - let full = bytes.len() / 4; - let tail = bytes.len() % 4; - - if tail == 1 { + let (groups, tail) = bytes.as_chunks::<4>(); + if tail.len() == 1 { return Err(PhcError::InvalidBase64); } - let expected_out = base64_decoded_len(bytes.len()); - if out.len() < expected_out { - return Err(PhcError::OutputBufferTooSmall); - } - - let mut written = 0usize; - for i in 0..full { - let off = i.strict_mul(4); - let d0 = B64_DECODE_TABLE[bytes[off] as usize]; - let d1 = B64_DECODE_TABLE[bytes[off.strict_add(1)] as usize]; - let d2 = B64_DECODE_TABLE[bytes[off.strict_add(2)] as usize]; - let d3 = B64_DECODE_TABLE[bytes[off.strict_add(3)] as usize]; + let destination = out.get_mut(..expected_out).ok_or(PhcError::OutputBufferTooSmall)?; + let (output_groups, output_tail) = destination.as_chunks_mut::<3>(); + + for (&[b0, b1, b2, b3], output) in groups.iter().zip(output_groups) { + let d0 = B64_DECODE_TABLE[usize::from(b0)]; + let d1 = B64_DECODE_TABLE[usize::from(b1)]; + let d2 = B64_DECODE_TABLE[usize::from(b2)]; + let d3 = B64_DECODE_TABLE[usize::from(b3)]; if (d0 | d1 | d2 | d3) == 0xFF { return Err(PhcError::InvalidBase64); } - let word = ((d0 as u32) << 18) | ((d1 as u32) << 12) | ((d2 as u32) << 6) | (d3 as u32); - out[written] = (word >> 16) as u8; - out[written.strict_add(1)] = (word >> 8) as u8; - out[written.strict_add(2)] = word as u8; - written = written.strict_add(3); + *output = [(d0 << 2) | (d1 >> 4), (d1 << 4) | (d2 >> 2), (d2 << 6) | d3]; } - let off = full.strict_mul(4); - match tail { - 0 => {} - 2 => { - let d0 = B64_DECODE_TABLE[bytes[off] as usize]; - let d1 = B64_DECODE_TABLE[bytes[off.strict_add(1)] as usize]; + match (tail, output_tail) { + ([], []) => {} + (&[b0, b1], [output]) => { + let d0 = B64_DECODE_TABLE[usize::from(b0)]; + let d1 = B64_DECODE_TABLE[usize::from(b1)]; if (d0 | d1) == 0xFF { return Err(PhcError::InvalidBase64); } @@ -148,13 +115,12 @@ pub(crate) fn base64_decode_into(s: &str, out: &mut [u8]) -> Result> 4); - written = written.strict_add(1); + *output = (d0 << 2) | (d1 >> 4); } - 3 => { - let d0 = B64_DECODE_TABLE[bytes[off] as usize]; - let d1 = B64_DECODE_TABLE[bytes[off.strict_add(1)] as usize]; - let d2 = B64_DECODE_TABLE[bytes[off.strict_add(2)] as usize]; + (&[b0, b1, b2], [output0, output1]) => { + let d0 = B64_DECODE_TABLE[usize::from(b0)]; + let d1 = B64_DECODE_TABLE[usize::from(b1)]; + let d2 = B64_DECODE_TABLE[usize::from(b2)]; if (d0 | d1 | d2) == 0xFF { return Err(PhcError::InvalidBase64); } @@ -162,15 +128,13 @@ pub(crate) fn base64_decode_into(s: &str, out: &mut [u8]) -> Result> 2); - out[written] = (word >> 8) as u8; - out[written.strict_add(1)] = word as u8; - written = written.strict_add(2); + *output0 = (d0 << 2) | (d1 >> 4); + *output1 = (d1 << 4) | (d2 >> 2); } _ => return Err(PhcError::InvalidBase64), } - Ok(written) + Ok(expected_out) } /// Append `n` as base-10 decimal (no leading zero) to `out`. @@ -189,12 +153,13 @@ pub(crate) fn push_u32_decimal(out: &mut String, n: u32) { let mut len = 0usize; let mut v = n; while v > 0 { - digits[len] = b'0' + (v % 10) as u8; + let [digit, _, _, _] = (v % 10).to_le_bytes(); + digits[len] = b'0'.wrapping_add(digit); v /= 10; len = len.strict_add(1); } for i in (0..len).rev() { - out.push(digits[i] as char); + out.push(char::from(digits[i])); } } @@ -206,15 +171,13 @@ pub(crate) fn push_u32_decimal(out: &mut String, n: u32) { /// keys, empty values, missing `=`, and empty pair segments are reported as /// `PhcError::MalformedParams`. pub(crate) struct PhcParamIter<'a> { - rest: &'a str, - done: bool, + rest: Option<&'a str>, } impl<'a> PhcParamIter<'a> { pub(crate) fn new(params: &'a str) -> Self { Self { - rest: params, - done: params.is_empty(), + rest: (!params.is_empty()).then_some(params), } } } @@ -223,35 +186,20 @@ impl<'a> Iterator for PhcParamIter<'a> { type Item = Result<(&'a str, &'a str), PhcError>; fn next(&mut self) -> Option { - if self.done { - return None; - } - let (pair, advance) = match self.rest.find(',') { - Some(idx) => { - // SAFETY: idx is a valid char boundary (',' is ASCII). - let pair = &self.rest[..idx]; - self.rest = &self.rest[idx.strict_add(1)..]; - (pair, false) - } - None => { - let pair = self.rest; - self.rest = ""; - (pair, true) - } + let rest = self.rest?; + let (pair, remaining) = match rest.split_once(',') { + Some((pair, remaining)) => (pair, Some(remaining)), + None => (rest, None), }; - if advance { - self.done = true; - } + self.rest = remaining; if pair.is_empty() { return Some(Err(PhcError::MalformedParams)); } - let eq = match pair.find('=') { - Some(i) => i, + let (key, value) = match pair.split_once('=') { + Some(fields) => fields, None => return Some(Err(PhcError::MalformedParams)), }; - let key = &pair[..eq]; - let value = &pair[eq.strict_add(1)..]; if key.is_empty() || value.is_empty() { return Some(Err(PhcError::MalformedParams)); } @@ -267,20 +215,21 @@ pub(crate) fn parse_param_u32(value: &str) -> Result { } // Reject leading zeros (e.g. "01") and leading sign (e.g. "-1", "+1"). let bytes = value.as_bytes(); - if bytes.len() > 1 && bytes[0] == b'0' { + if matches!(bytes, [b'0', _, ..]) { return Err(PhcError::MalformedParams); } - let mut acc: u64 = 0; + let mut acc = 0u32; for &b in bytes { if !b.is_ascii_digit() { return Err(PhcError::MalformedParams); } - acc = acc.strict_mul(10).strict_add((b - b'0') as u64); - if acc > u32::MAX as u64 { - return Err(PhcError::ParamOutOfRange); - } + let digit = u32::from(b.wrapping_sub(b'0')); + acc = acc + .checked_mul(10) + .and_then(|prefix| prefix.checked_add(digit)) + .ok_or(PhcError::ParamOutOfRange)?; } - Ok(u32::try_from(acc).unwrap_or_else(|_| unreachable!("acc <= u32::MAX, enforced inside the loop"))) + Ok(acc) } // ─── Segmented PHC parser ─────────────────────────────────────────────────── @@ -440,13 +389,13 @@ mod tests { #[test] fn base64_roundtrip_all_lengths_0_to_64() { - for len in 0..=64 { - let input: Vec = (0..len).map(|i| ((i * 31 + 7) & 0xff) as u8).collect(); + for len in 0u8..=64 { + let input: Vec = (0..len).map(|i| i.wrapping_mul(31).wrapping_add(7)).collect(); let mut encoded = String::new(); base64_encode_into(&input, &mut encoded); let mut decoded = vec![0u8; base64_decoded_len(encoded.len())]; - let n = base64_decode_into(&encoded, &mut decoded).unwrap(); + let n = base64_decode_into(&encoded, &mut decoded).expect("encoder output must be canonical base64"); decoded.truncate(n); assert_eq!(decoded, input, "roundtrip failed at len={len}"); } @@ -478,6 +427,13 @@ mod tests { let mut out = [0u8; 32]; assert_eq!(base64_decode_into("A", &mut out), Err(PhcError::InvalidBase64)); assert_eq!(base64_decode_into("AAAAA", &mut out), Err(PhcError::InvalidBase64)); + + let mut empty = []; + assert_eq!( + base64_decode_into("A", &mut empty), + Err(PhcError::InvalidBase64), + "an invalid tail takes precedence over destination sizing" + ); } #[test] @@ -506,16 +462,26 @@ mod tests { #[test] fn param_iter_single_pair() { let mut it = PhcParamIter::new("m=65536"); - assert_eq!(it.next().unwrap().unwrap(), ("m", "65536")); + assert_eq!( + it.next() + .expect("one parameter must be present") + .expect("the parameter must be well-formed"), + ("m", "65536") + ); assert!(it.next().is_none()); } #[test] fn param_iter_multiple_pairs() { let mut it = PhcParamIter::new("m=65536,t=3,p=4"); - assert_eq!(it.next().unwrap().unwrap(), ("m", "65536")); - assert_eq!(it.next().unwrap().unwrap(), ("t", "3")); - assert_eq!(it.next().unwrap().unwrap(), ("p", "4")); + for expected in [("m", "65536"), ("t", "3"), ("p", "4")] { + assert_eq!( + it.next() + .expect("the expected parameter must be present") + .expect("the parameter must be well-formed"), + expected + ); + } assert!(it.next().is_none()); } @@ -528,34 +494,69 @@ mod tests { #[test] fn param_iter_rejects_missing_equals() { let mut it = PhcParamIter::new("mX65536"); - assert_eq!(it.next().unwrap(), Err(PhcError::MalformedParams)); + assert_eq!( + it.next().expect("the malformed parameter must be emitted"), + Err(PhcError::MalformedParams) + ); } #[test] fn param_iter_rejects_empty_pair_segment() { let mut it = PhcParamIter::new("m=1,,p=2"); - assert_eq!(it.next().unwrap().unwrap(), ("m", "1")); - assert_eq!(it.next().unwrap(), Err(PhcError::MalformedParams)); + assert_eq!( + it.next() + .expect("the first parameter must be present") + .expect("the first parameter must be well-formed"), + ("m", "1") + ); + assert_eq!( + it.next().expect("the empty parameter must be emitted"), + Err(PhcError::MalformedParams) + ); + } + + #[test] + fn param_iter_rejects_trailing_empty_pair_segment() { + let mut it = PhcParamIter::new("m=1,"); + assert_eq!( + it.next() + .expect("the first parameter must be present") + .expect("the first parameter must be well-formed"), + ("m", "1") + ); + assert_eq!( + it.next().expect("the trailing empty parameter must be emitted"), + Err(PhcError::MalformedParams) + ); + assert!(it.next().is_none()); } #[test] fn param_iter_rejects_empty_key() { let mut it = PhcParamIter::new("=65536"); - assert_eq!(it.next().unwrap(), Err(PhcError::MalformedParams)); + assert_eq!( + it.next().expect("the malformed parameter must be emitted"), + Err(PhcError::MalformedParams) + ); } #[test] fn param_iter_rejects_empty_value() { let mut it = PhcParamIter::new("m="); - assert_eq!(it.next().unwrap(), Err(PhcError::MalformedParams)); + assert_eq!( + it.next().expect("the malformed parameter must be emitted"), + Err(PhcError::MalformedParams) + ); } #[test] fn parse_param_u32_accepts_valid() { - assert_eq!(parse_param_u32("0").unwrap(), 0); - assert_eq!(parse_param_u32("1").unwrap(), 1); - assert_eq!(parse_param_u32("65536").unwrap(), 65_536); - assert_eq!(parse_param_u32("4294967295").unwrap(), u32::MAX); + for (encoded, expected) in [("0", 0), ("1", 1), ("65536", 65_536), ("4294967295", u32::MAX)] { + assert_eq!( + parse_param_u32(encoded).expect("canonical u32 text must parse"), + expected + ); + } } #[test] @@ -574,7 +575,7 @@ mod tests { #[test] fn parse_argon2id_canonical() { let encoded = "$argon2id$v=19$m=65536,t=3,p=4$c29tZXNhbHQ$c29tZWhhc2g"; - let parts = parse(encoded).unwrap(); + let parts = parse(encoded).expect("canonical Argon2id PHC text must parse"); assert_eq!(parts.algorithm, "argon2id"); assert_eq!(parts.version, Some("19")); assert_eq!(parts.parameters, "m=65536,t=3,p=4"); @@ -585,7 +586,7 @@ mod tests { #[test] fn parse_scrypt_no_version() { let encoded = "$scrypt$ln=14,r=8,p=1$c29tZXNhbHQ$c29tZWhhc2g"; - let parts = parse(encoded).unwrap(); + let parts = parse(encoded).expect("canonical scrypt PHC text must parse"); assert_eq!(parts.algorithm, "scrypt"); assert_eq!(parts.version, None); assert_eq!(parts.parameters, "ln=14,r=8,p=1"); @@ -636,7 +637,7 @@ mod tests { #[test] fn parse_without_version_segment_returns_none() { - let parts = parse("$argon2id$m=1,t=1,p=1$c29tZQ$c29tZQ").unwrap(); + let parts = parse("$argon2id$m=1,t=1,p=1$c29tZQ$c29tZQ").expect("PHC text without a version segment must parse"); assert_eq!(parts.version, None); } diff --git a/src/auth/poly1305.rs b/src/auth/poly1305.rs index b8bee637..06f8ddc2 100644 --- a/src/auth/poly1305.rs +++ b/src/auth/poly1305.rs @@ -1,7 +1,5 @@ //! Standalone Poly1305 one-time authenticator (RFC 8439). -#![allow(clippy::indexing_slicing)] // Poly1305 uses fixed 16-byte block and limb offsets. - use core::fmt; use crate::{ @@ -22,6 +20,22 @@ fn load_u32_le(input: &[u8]) -> u32 { u32::from_le_bytes(bytes) } +#[inline(always)] +fn low_u32(value: u64) -> u32 { + let bytes = value.to_le_bytes(); + u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) +} + +#[inline(always)] +fn limb_product(left: u32, right: u32) -> u64 { + u64::from(left).wrapping_mul(u64::from(right)) +} + +#[inline(always)] +fn limb_multiply_accumulate(accumulator: u64, left: u32, right: u32) -> u64 { + accumulator.wrapping_add(limb_product(left, right)) +} + /// Poly1305 one-time key. /// /// This type is intentionally not `Clone` or `Copy`. Poly1305 keys must be @@ -231,10 +245,10 @@ impl State { let r3 = self.r[3]; let r4 = self.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = r1.wrapping_mul(5); + let s2 = r2.wrapping_mul(5); + let s3 = r3.wrapping_mul(5); + let s4 = r4.wrapping_mul(5); let mut h0 = self.h[0]; let mut h1 = self.h[1]; @@ -248,51 +262,55 @@ impl State { h3 = h3.wrapping_add((load_u32_le(&block[9..13]) >> 6) & LIMB_MASK); h4 = h4.wrapping_add((load_u32_le(&block[12..16]) >> 8) | hibit); - let d0 = (u64::from(h0) * u64::from(r0)) - + (u64::from(h1) * u64::from(s4)) - + (u64::from(h2) * u64::from(s3)) - + (u64::from(h3) * u64::from(s2)) - + (u64::from(h4) * u64::from(s1)); - let mut d1 = (u64::from(h0) * u64::from(r1)) - + (u64::from(h1) * u64::from(r0)) - + (u64::from(h2) * u64::from(s4)) - + (u64::from(h3) * u64::from(s3)) - + (u64::from(h4) * u64::from(s2)); - let mut d2 = (u64::from(h0) * u64::from(r2)) - + (u64::from(h1) * u64::from(r1)) - + (u64::from(h2) * u64::from(r0)) - + (u64::from(h3) * u64::from(s4)) - + (u64::from(h4) * u64::from(s3)); - let mut d3 = (u64::from(h0) * u64::from(r3)) - + (u64::from(h1) * u64::from(r2)) - + (u64::from(h2) * u64::from(r1)) - + (u64::from(h3) * u64::from(r0)) - + (u64::from(h4) * u64::from(s4)); - let mut d4 = (u64::from(h0) * u64::from(r4)) - + (u64::from(h1) * u64::from(r3)) - + (u64::from(h2) * u64::from(r2)) - + (u64::from(h3) * u64::from(r1)) - + (u64::from(h4) * u64::from(r0)); - - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); - - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); - - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); - - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); - - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + let mut d0 = limb_product(h0, r0); + d0 = limb_multiply_accumulate(d0, h1, s4); + d0 = limb_multiply_accumulate(d0, h2, s3); + d0 = limb_multiply_accumulate(d0, h3, s2); + d0 = limb_multiply_accumulate(d0, h4, s1); + + let mut d1 = limb_product(h0, r1); + d1 = limb_multiply_accumulate(d1, h1, r0); + d1 = limb_multiply_accumulate(d1, h2, s4); + d1 = limb_multiply_accumulate(d1, h3, s3); + d1 = limb_multiply_accumulate(d1, h4, s2); + + let mut d2 = limb_product(h0, r2); + d2 = limb_multiply_accumulate(d2, h1, r1); + d2 = limb_multiply_accumulate(d2, h2, r0); + d2 = limb_multiply_accumulate(d2, h3, s4); + d2 = limb_multiply_accumulate(d2, h4, s3); + + let mut d3 = limb_product(h0, r3); + d3 = limb_multiply_accumulate(d3, h1, r2); + d3 = limb_multiply_accumulate(d3, h2, r1); + d3 = limb_multiply_accumulate(d3, h3, r0); + d3 = limb_multiply_accumulate(d3, h4, s4); + + let mut d4 = limb_product(h0, r4); + d4 = limb_multiply_accumulate(d4, h1, r3); + d4 = limb_multiply_accumulate(d4, h2, r2); + d4 = limb_multiply_accumulate(d4, h3, r1); + d4 = limb_multiply_accumulate(d4, h4, r0); + + let mut c = low_u32(d0 >> 26); + h0 = low_u32(d0) & LIMB_MASK; + d1 = d1.wrapping_add(u64::from(c)); + + c = low_u32(d1 >> 26); + h1 = low_u32(d1) & LIMB_MASK; + d2 = d2.wrapping_add(u64::from(c)); + + c = low_u32(d2 >> 26); + h2 = low_u32(d2) & LIMB_MASK; + d3 = d3.wrapping_add(u64::from(c)); + + c = low_u32(d3 >> 26); + h3 = low_u32(d3) & LIMB_MASK; + d4 = d4.wrapping_add(u64::from(c)); + + c = low_u32(d4 >> 26); + h4 = low_u32(d4) & LIMB_MASK; + h0 = h0.wrapping_add(c.wrapping_mul(5)); c = h0 >> 26; h0 &= LIMB_MASK; @@ -323,7 +341,7 @@ impl State { c = h4 >> 26; h4 &= LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + h0 = h0.wrapping_add(c.wrapping_mul(5)); c = h0 >> 26; h0 &= LIMB_MASK; @@ -366,14 +384,14 @@ impl State { h2 = (h2 >> 12) | (h3 << 14); h3 = (h3 >> 18) | (h4 << 8); - let mut f = u64::from(h0) + u64::from(self.pad[0]); - h0 = f as u32; - f = u64::from(h1) + u64::from(self.pad[1]) + (f >> 32); - h1 = f as u32; - f = u64::from(h2) + u64::from(self.pad[2]) + (f >> 32); - h2 = f as u32; - f = u64::from(h3) + u64::from(self.pad[3]) + (f >> 32); - h3 = f as u32; + let mut f = u64::from(h0).wrapping_add(u64::from(self.pad[0])); + h0 = low_u32(f); + f = u64::from(h1).wrapping_add(u64::from(self.pad[1])).wrapping_add(f >> 32); + h1 = low_u32(f); + f = u64::from(h2).wrapping_add(u64::from(self.pad[2])).wrapping_add(f >> 32); + h2 = low_u32(f); + f = u64::from(h3).wrapping_add(u64::from(self.pad[3])).wrapping_add(f >> 32); + h3 = low_u32(f); let mut tag = [0u8; TAG_SIZE]; tag[0..4].copy_from_slice(&h0.to_le_bytes()); @@ -433,7 +451,7 @@ impl Poly1305 { #[inline] pub fn update(&mut self, mut data: &[u8]) { if self.buffer_len != 0 { - let take = core::cmp::min(TAG_SIZE - self.buffer_len, data.len()); + let take = core::cmp::min(TAG_SIZE.strict_sub(self.buffer_len), data.len()); self.buffer[self.buffer_len..self.buffer_len.strict_add(take)].copy_from_slice(&data[..take]); self.buffer_len = self.buffer_len.strict_add(take); data = &data[take..]; @@ -445,15 +463,11 @@ impl Poly1305 { } } - let mut blocks = data.chunks_exact(TAG_SIZE); - for chunk in &mut blocks { - let mut block = [0u8; TAG_SIZE]; - block.copy_from_slice(chunk); - self.state.compute_block(&block, false); - ct::zeroize_no_fence(&mut block); + let (blocks, rem) = data.as_chunks::(); + for block in blocks { + self.state.compute_block(block, false); } - let rem = blocks.remainder(); if !rem.is_empty() { self.buffer[..rem.len()].copy_from_slice(rem); self.buffer_len = rem.len(); @@ -512,7 +526,7 @@ impl Drop for Poly1305 { fn drop(&mut self) { ct::zeroize(&mut self.buffer); // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.buffer_len, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.buffer_len, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } diff --git a/src/auth/rsa.rs b/src/auth/rsa.rs index da035f87..b3214b18 100644 --- a/src/auth/rsa.rs +++ b/src/auth/rsa.rs @@ -1275,6 +1275,21 @@ pub struct RsaPrivateKeyParts<'a> { pub coefficient: &'a [u8], } +impl fmt::Debug for RsaPrivateKeyParts<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RsaPrivateKeyParts") + .field("modulus_bits", &unsigned_be_bit_len(self.modulus)) + .field("public_exponent", &self.public_exponent) + .field("private_exponent", &"****") + .field("prime_p", &"****") + .field("prime_q", &"****") + .field("exponent_p", &"****") + .field("exponent_q", &"****") + .field("coefficient", &"****") + .finish() + } +} + impl RsaPrivateKey { /// Key-generation contract used by [`Self::generate`] and /// [`Self::generate_with_policy`]. @@ -2053,11 +2068,7 @@ impl RsaPrivateKey { clear_decryption_output_on_error(result, out) } - /// Sign a message using RSASSA-PKCS1-v1_5 with a caller-supplied blinding factor. - /// - /// `blinding_factor` and `blinding_factor_inverse` must be fixed-width - /// modulus-sized representatives satisfying - /// `blinding_factor * blinding_factor_inverse == 1 mod n`. + /// Sign a message using RSASSA-PKCS1-v1_5 with caller-supplied blinding. /// /// # Errors /// @@ -2068,23 +2079,19 @@ impl RsaPrivateKey { &self, profile: RsaPkcs1v15Profile, message: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], ) -> Result<(), RsaPrivateOpError> { - let result = self.components.sign_pkcs1v15_with_blinding_factor( - profile, - message, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - ); + let result = self + .components + .sign_pkcs1v15_with_blinding_factor(profile, message, blinding, out); clear_output_on_error(result, out) } /// Sign using RSASSA-PKCS1-v1_5 with caller-supplied blinding and scratch. /// /// Reusing scratch avoids top-level private-operation temporary allocation - /// after setup. The blinding-factor requirements are the same as + /// after setup. The blinding-pair requirements are the same as /// [`Self::sign_pkcs1v15_with_blinding_factor`]. /// /// # Errors @@ -2097,18 +2104,13 @@ impl RsaPrivateKey { &self, profile: RsaPkcs1v15Profile, message: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], scratch: &mut RsaPrivateScratch, ) -> Result<(), RsaPrivateOpError> { - let result = self.components.sign_pkcs1v15_with_blinding_factor_and_scratch( - profile, - message, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - scratch, - ); + let result = self + .components + .sign_pkcs1v15_with_blinding_factor_and_scratch(profile, message, blinding, out, scratch); clear_output_on_error(result, out) } @@ -2125,24 +2127,19 @@ impl RsaPrivateKey { profile: RsaPssProfile, message: &[u8], salt: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], ) -> Result<(), RsaPrivateOpError> { - let result = self.components.sign_pss_with_salt_and_blinding_factor( - profile, - message, - salt, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - ); + let result = self + .components + .sign_pss_with_salt_and_blinding_factor(profile, message, salt, blinding, out); clear_output_on_error(result, out) } /// Sign using RSASSA-PSS with explicit salt, caller blinding, and scratch. /// /// Reusing scratch avoids top-level private-operation temporary allocation - /// after setup. The salt and blinding-factor requirements are the same as + /// after setup. The salt and blinding-pair requirements are the same as /// [`Self::sign_pss_with_salt_and_blinding_factor`]. /// /// # Errors @@ -2151,26 +2148,19 @@ impl RsaPrivateKey { /// allocated for a different modulus width, the blinding pair is invalid, the /// key is too small for the salt/profile, or the post-signing public fault /// check fails. - #[allow(clippy::too_many_arguments)] #[must_use = "RSA signing failure must be checked; a dropped Result silently discards a failed signature"] pub fn sign_pss_with_salt_and_blinding_factor_and_scratch( &self, profile: RsaPssProfile, message: &[u8], salt: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], scratch: &mut RsaPrivateScratch, ) -> Result<(), RsaPrivateOpError> { - let result = self.components.sign_pss_with_salt_and_blinding_factor_and_scratch( - profile, - message, - salt, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - scratch, - ); + let result = self + .components + .sign_pss_with_salt_and_blinding_factor_and_scratch(profile, message, salt, blinding, out, scratch); clear_output_on_error(result, out) } @@ -2190,17 +2180,12 @@ impl RsaPrivateKey { profile: RsaOaepProfile, label: &[u8], ciphertext: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], ) -> Result { - let result = self.components.decrypt_oaep_with_blinding_factor( - profile, - label, - ciphertext, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - ); + let result = self + .components + .decrypt_oaep_with_blinding_factor(profile, label, ciphertext, blinding, out); clear_decryption_output_on_error(result, out) } @@ -2218,22 +2203,19 @@ impl RsaPrivateKey { pub fn decrypt_pkcs1v15_with_blinding_factor( &self, ciphertext: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], ) -> Result { - let result = self.components.decrypt_pkcs1v15_with_blinding_factor( - ciphertext, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - ); + let result = self + .components + .decrypt_pkcs1v15_with_blinding_factor(ciphertext, blinding, out); clear_decryption_output_on_error(result, out) } /// Decrypt RSAES-OAEP with caller-supplied blinding and scratch. /// /// Reusing scratch avoids top-level private-operation temporary allocation - /// after setup. The blinding-factor requirements are the same as + /// after setup. The blinding-pair requirements are the same as /// [`Self::decrypt_oaep_with_blinding_factor`]. /// /// # Errors @@ -2241,33 +2223,26 @@ impl RsaPrivateKey { /// Returns [`RsaPrivateOpError`] if lengths are invalid, `scratch` was /// allocated for a different modulus width, the blinding pair is invalid, the /// ciphertext representative is out of range, or OAEP decoding fails. - #[allow(clippy::too_many_arguments)] #[must_use = "RSA decryption failure must be checked; a dropped Result silently discards plaintext"] pub fn decrypt_oaep_with_blinding_factor_and_scratch( &self, profile: RsaOaepProfile, label: &[u8], ciphertext: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], scratch: &mut RsaPrivateScratch, ) -> Result { - let result = self.components.decrypt_oaep_with_blinding_factor_and_scratch( - profile, - label, - ciphertext, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - scratch, - ); + let result = self + .components + .decrypt_oaep_with_blinding_factor_and_scratch(profile, label, ciphertext, blinding, out, scratch); clear_decryption_output_on_error(result, out) } /// Decrypt RSAES-PKCS1-v1_5 with caller-supplied blinding and scratch. /// /// Reusing scratch avoids top-level private-operation temporary allocation - /// after setup. The blinding-factor requirements are the same as + /// after setup. The blinding-pair requirements are the same as /// [`Self::decrypt_pkcs1v15_with_blinding_factor`]. /// /// # Errors @@ -2279,17 +2254,13 @@ impl RsaPrivateKey { pub fn decrypt_pkcs1v15_with_blinding_factor_and_scratch( &self, ciphertext: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], scratch: &mut RsaPrivateScratch, ) -> Result { - let result = self.components.decrypt_pkcs1v15_with_blinding_factor_and_scratch( - ciphertext, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - scratch, - ); + let result = self + .components + .decrypt_pkcs1v15_with_blinding_factor_and_scratch(ciphertext, blinding, out, scratch); clear_decryption_output_on_error(result, out) } } @@ -2612,7 +2583,6 @@ impl Drop for RsaPrivateScratch { // separate from private-operation exponentiation so validation arithmetic does // not become signing, decryption, blinding, CRT recombination, or key // generation arithmetic by accident. -#[allow(dead_code)] struct RsaPrivateKeyComponents { public: RsaPublicKey, private_exponent: SecretBigEndianInteger, @@ -2625,7 +2595,6 @@ struct RsaPrivateKeyComponents { coefficient: SecretBigEndianInteger, } -#[allow(dead_code)] impl RsaPrivateKeyComponents { #[inline] fn public_key(&self) -> &RsaPublicKey { @@ -2991,6 +2960,7 @@ impl RsaPrivateKeyComponents { result } + #[cfg(feature = "getrandom")] fn sign_pkcs1v15_with_stored_blinding_and_scratch( &self, profile: RsaPkcs1v15Profile, @@ -3084,6 +3054,7 @@ impl RsaPrivateKeyComponents { result } + #[cfg(feature = "getrandom")] fn sign_pss_with_stored_salt_and_blinding_and_scratch( &self, profile: RsaPssProfile, @@ -3162,6 +3133,7 @@ impl RsaPrivateKeyComponents { self.private_operation_with_blinding_factor_and_scratch(blinding, out, scratch) } + #[cfg(feature = "getrandom")] fn sign_encoded_message_with_stored_blinding_and_scratch( &self, out: &mut [u8], @@ -3213,6 +3185,7 @@ impl RsaPrivateKeyComponents { clear_decryption_output_on_error(result, out) } + #[cfg(feature = "getrandom")] fn decrypt_oaep_with_stored_blinding_and_scratch( &self, profile: RsaOaepProfile, @@ -3262,6 +3235,7 @@ impl RsaPrivateKeyComponents { clear_decryption_output_on_error(result, out) } + #[cfg(feature = "getrandom")] fn decrypt_pkcs1v15_with_stored_blinding_and_scratch( &self, ciphertext: &[u8], @@ -3457,6 +3431,7 @@ impl RsaPrivateKeyComponents { } } + #[cfg(feature = "getrandom")] fn private_operation_from_scratch_encoded_with_stored_blinding( &self, scratch: &mut RsaPrivateScratch, @@ -3690,12 +3665,10 @@ impl fmt::Debug for RsaPrivateKeyComponents { } } -#[allow(dead_code)] struct SecretBigEndianInteger { bytes: Box<[u8]>, } -#[allow(dead_code)] impl SecretBigEndianInteger { fn new(bytes: &[u8]) -> Result { if is_zero_unsigned_be(bytes) { @@ -3706,6 +3679,7 @@ impl SecretBigEndianInteger { }) } + #[cfg(feature = "getrandom")] fn from_vec(mut bytes: Vec) -> Result { if is_zero_unsigned_be(&bytes) { ct::zeroize(&mut bytes); @@ -3804,29 +3778,50 @@ impl RsaBlindingFactor { } } -struct RsaBlindingPair<'a> { +/// Borrowed inputs for a caller-blinded RSA private operation. +/// +/// The factor and inverse must be fixed-width, modulus-sized representatives +/// satisfying `factor * inverse == 1 mod n`. Construction only borrows the +/// inputs; each private operation validates them against its key before secret +/// exponentiation. This type does not own, copy, or zeroize the borrowed bytes. +#[derive(Clone, Copy)] +pub struct RsaBlindingPair<'a> { factor: &'a [u8], inverse: &'a [u8], validate: bool, } +impl fmt::Debug for RsaBlindingPair<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RsaBlindingPair") + .field("factor", &"****") + .field("inverse", &"****") + .finish() + } +} + impl<'a> RsaBlindingPair<'a> { - #[cfg(feature = "getrandom")] + /// Borrow a factor and its claimed modular inverse. + /// + /// Validation requires the target RSA modulus and is therefore deferred to + /// the private operation that consumes this value. #[inline] - const fn trusted(factor: &'a [u8], inverse: &'a [u8]) -> Self { + #[must_use] + pub const fn new(factor: &'a [u8], inverse: &'a [u8]) -> Self { Self { factor, inverse, - validate: false, + validate: true, } } + #[cfg(feature = "getrandom")] #[inline] - const fn caller_supplied(factor: &'a [u8], inverse: &'a [u8]) -> Self { + const fn trusted(factor: &'a [u8], inverse: &'a [u8]) -> Self { Self { factor, inverse, - validate: true, + validate: false, } } } @@ -5768,11 +5763,8 @@ impl RsaPublicModulus { } } - fn montgomery_r2(&self) -> &[u64] { - match self.r2.as_deref() { - Some(r2) => r2, - None => unreachable!("private RSA modulus missing Montgomery R^2"), - } + fn montgomery_r2(&self) -> Option<&[u64]> { + self.r2.as_deref() } fn public_operation( @@ -5831,7 +5823,7 @@ impl RsaPublicModulus { && rsa_aarch64_asm::supports_bignum_mont_words(limbs) && t.len() >= rsa_aarch64_asm::bignum_mont_scratch_words(limbs) { - rsa_aarch64_asm::public_e65537_mont_words(base, x, r2, acc, &self.limbs, self.n0, limbs, t); + rsa_aarch64_asm::public_e65537_mont_words(base, x, r2, acc, &self.limbs, self.n0, t); limbs_to_be(base, out); return Ok(()); } @@ -5846,7 +5838,7 @@ impl RsaPublicModulus { && rsa_aarch64_linux_asm::supports_bignum_mont_words(limbs) && t.len() >= rsa_aarch64_linux_asm::bignum_mont_scratch_words(limbs) { - rsa_aarch64_linux_asm::public_e65537_mont_words(base, x, r2, acc, &self.limbs, self.n0, limbs, t); + rsa_aarch64_linux_asm::public_e65537_mont_words(base, x, r2, acc, &self.limbs, self.n0, t); limbs_to_be(base, out); return Ok(()); } @@ -6463,23 +6455,20 @@ fn der_len_len(len: usize) -> usize { 1usize.strict_add(significant) } -#[allow(clippy::cast_possible_truncation)] fn der_push_len(len: usize, out: &mut Vec) { if len < 128 { - out.push(len as u8); + out.push(u8::try_from(len).expect("a short DER length must fit in one byte")); return; } - let mut started = false; - let len_len = der_len_len(len).strict_sub(1); - out.push(0x80 | (len_len as u8)); - for index in (0..core::mem::size_of::()).rev() { - let byte = (len >> index.strict_mul(8)) as u8; - if byte != 0 || started { - out.push(byte); - started = true; - } - } + let bytes = len.to_be_bytes(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .unwrap_or_else(|| bytes.len().strict_sub(1)); + let len_bytes = &bytes[first_nonzero..]; + out.push(0x80 | u8::try_from(len_bytes.len()).expect("a usize DER length prefix must fit in one byte")); + out.extend_from_slice(len_bytes); } fn parse_rsa_algorithm_identifier(der: &[u8]) -> Result<(), RsaKeyError> { @@ -6496,7 +6485,6 @@ fn parse_rsa_algorithm_identifier(der: &[u8]) -> Result<(), RsaKeyError> { reader.finish() } -#[allow(dead_code)] fn parse_pkcs8_private_key_der_with_policy( der: &[u8], policy: &RsaPublicKeyPolicy, @@ -6522,7 +6510,6 @@ fn parse_pkcs8_private_key_der_parts_with_policy<'a>( parse_pkcs1_private_key_der_parts_with_policy(private_key, policy) } -#[allow(dead_code)] fn parse_pkcs1_private_key_der_with_policy( der: &[u8], policy: &RsaPublicKeyPolicy, @@ -7292,7 +7279,7 @@ fn keygen_mask_unused_top_bits(bytes: &mut [u8], bits: usize) { return; } if let Some(first) = bytes.first_mut() { - *first &= (1u8 << used_top_bits) - 1; + *first &= u8::MAX >> 8usize.strict_sub(used_top_bits); } } @@ -7596,7 +7583,10 @@ fn keygen_inverse_small_mod_odd(exponent: u64, modulus: &[u8]) -> Result } for k in 1..exponent { - if (1u128 + u128::from(k).strict_mul(u128::from(modulus_mod_exponent))).is_multiple_of(u128::from(exponent)) { + if 1u128 + .strict_add(u128::from(k).strict_mul(u128::from(modulus_mod_exponent))) + .is_multiple_of(u128::from(exponent)) + { return keygen_mul_u64_add_one_div_u64(modulus, k, exponent); } } @@ -7614,11 +7604,11 @@ fn keygen_mul_u64_add_one_div_u64( let mut carry = 1u128; for &byte in value.iter().rev() { let acc = u128::from(byte).strict_mul(u128::from(multiplier)).strict_add(carry); - product_rev.push(acc as u8); + product_rev.push(acc.to_le_bytes()[0]); carry = acc >> 8; } while carry != 0 { - product_rev.push(carry as u8); + product_rev.push(carry.to_le_bytes()[0]); carry >>= 8; } product_rev.reverse(); @@ -7628,8 +7618,8 @@ fn keygen_mul_u64_add_one_div_u64( let divisor = u128::from(divisor); for &byte in &product_rev { let acc = (remainder << 8).strict_add(u128::from(byte)); - quotient.push((acc / divisor) as u8); - remainder = acc % divisor; + quotient.push(u8::try_from(acc.div_euclid(divisor)).map_err(|_| RsaKeyGenerationError::ArithmeticFailure)?); + remainder = acc.rem_euclid(divisor); } ct::zeroize(&mut product_rev); if remainder != 0 { @@ -7699,9 +7689,9 @@ fn unsigned_be_mod_u64(bytes: &[u8], modulus: u64) -> u64 { let mut remainder = 0u128; let modulus = u128::from(modulus); for &byte in bytes { - remainder = ((remainder << 8).strict_add(u128::from(byte))) % modulus; + remainder = (remainder << 8).strict_add(u128::from(byte)).rem_euclid(modulus); } - remainder as u64 + split_u128(remainder).0 } const fn tls13_signature_scheme_profile_and_key_algorithm( @@ -8102,7 +8092,6 @@ fn validate_modulus(modulus: &[u8], policy: &RsaPublicKeyPolicy) -> Result(message: &[u8], digest_info_prefix: &[u8], out: &mut [u8]) -> Result<(), RsaPrivateOpError> where D: Digest, @@ -8154,7 +8143,6 @@ fn fill_pkcs1v15_nonzero_padding_with( Ok(()) } -#[allow(clippy::indexing_slicing)] fn encode_pkcs1v15_encryption_with_seed(message: &[u8], seed: &[u8], out: &mut [u8]) -> Result<(), RsaEncryptionError> { let ps_len = pkcs1v15_encryption_padding_len(out.len(), message.len())?; if seed.len() != ps_len { @@ -8222,7 +8210,6 @@ fn clear_output_on_error(result: Result, out: &mut [u8]) -> Result(message: &[u8], salt: &[u8], em_bits: usize, out: &mut [u8]) -> Result<(), RsaPrivateOpError> where D: Digest, @@ -8239,7 +8226,6 @@ where result } -#[allow(clippy::indexing_slicing)] fn encode_pss_with_mask( message: &[u8], salt: &[u8], @@ -8301,7 +8287,6 @@ where Ok(()) } -#[allow(clippy::indexing_slicing)] fn encode_oaep_with_masks( label: &[u8], message: &[u8], @@ -8353,7 +8338,6 @@ where Ok(()) } -#[allow(clippy::indexing_slicing)] fn decode_oaep(label: &[u8], encoded: &mut [u8], out: &mut [u8]) -> Result where D: Digest, @@ -8372,7 +8356,6 @@ where result } -#[allow(clippy::indexing_slicing)] fn decode_oaep_with_masks( label: &[u8], encoded: &mut [u8], @@ -8491,7 +8474,7 @@ where return Err(VerificationError::new()); } if unused_bits > 0 { - let mask = 0xffu8 << (8usize.strict_sub(unused_bits) as u32); + let mask = 0xffu8 << 8usize.strict_sub(unused_bits); if masked_db.first().copied().unwrap_or(0) & mask != 0 { return Err(VerificationError::new()); } @@ -8589,7 +8572,6 @@ where } } -#[allow(clippy::indexing_slicing)] fn limbs_from_be(bytes: &[u8]) -> Vec { let limbs = bytes.len().strict_add(7) / 8; let mut out = vec![0u64; limbs]; @@ -8597,11 +8579,10 @@ fn limbs_from_be(bytes: &[u8]) -> Vec { out } -#[allow(clippy::indexing_slicing)] fn limbs_from_be_into(bytes: &[u8], out: &mut [u64]) { let full_limbs = bytes.len() / 8; let leading = bytes.len() % 8; - let needed_limbs = full_limbs + usize::from(leading != 0); + let needed_limbs = bytes.len().div_ceil(8); if out.len() != needed_limbs { out.fill(0); } @@ -8622,7 +8603,6 @@ fn limbs_from_be_into(bytes: &[u8], out: &mut [u64]) { } } -#[allow(clippy::indexing_slicing)] fn limbs_to_be(limbs: &[u64], out: &mut [u8]) { let full_limbs = out.len() / 8; let leading = out.len() % 8; @@ -8741,7 +8721,6 @@ fn left_pad_be(src: &[u8], out: &mut [u8]) -> Result<(), RsaPrivateOpError> { Ok(()) } -#[allow(clippy::indexing_slicing)] fn private_sub_mod_unsigned_be_to_fixed( left: &[u8], right: &[u8], @@ -8757,7 +8736,7 @@ fn private_sub_mod_unsigned_be_to_fixed( let src = out.len().strict_sub(index).strict_sub(1); let subtrahend = u16::from(right[src]).strict_add(borrow); let difference = u16::from(left[src]).wrapping_sub(subtrahend); - out[src] = difference as u8; + out[src] = difference.to_le_bytes()[0]; borrow = (difference >> 15) & 1; } @@ -8767,14 +8746,13 @@ fn private_sub_mod_unsigned_be_to_fixed( let dst = out.len().strict_sub(index).strict_sub(1); let addend = u16::from(modulus[dst]) & mask; let sum = u16::from(out[dst]).strict_add(addend).strict_add(carry); - out[dst] = sum as u8; + out[dst] = sum.to_le_bytes()[0]; carry = sum >> 8; } Ok(()) } -#[allow(clippy::indexing_slicing)] #[cfg(feature = "getrandom")] fn private_sub_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: &mut [u8]) -> Result<(), RsaPrivateOpError> { if left.len() != right.len() || left.len() != out.len() || unsigned_be_cmp(left, right) == core::cmp::Ordering::Less { @@ -8784,14 +8762,16 @@ fn private_sub_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: &mut [u8]) - let mut borrow = 0i16; for index in 0..out.len() { let src = out.len().strict_sub(index).strict_sub(1); - let mut difference = i16::from(left[src]) - i16::from(right[src]) - borrow; + let mut difference = i16::from(left[src]) + .strict_sub(i16::from(right[src])) + .strict_sub(borrow); if difference < 0 { - difference += 256; + difference = difference.strict_add(256); borrow = 1; } else { borrow = 0; } - out[src] = difference as u8; + out[src] = difference.to_le_bytes()[0]; } if borrow == 0 { @@ -8801,7 +8781,6 @@ fn private_sub_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: &mut [u8]) - } } -#[allow(clippy::indexing_slicing)] #[cfg(feature = "getrandom")] fn private_sub_unsigned_be_to_len( left: &[u8], @@ -8827,15 +8806,17 @@ fn private_sub_unsigned_be_to_len( .and_then(|src| right.get(src)) .copied() .unwrap_or(0); - let mut difference = i16::from(left_byte) - i16::from(right_byte) - borrow; + let mut difference = i16::from(left_byte) + .strict_sub(i16::from(right_byte)) + .strict_sub(borrow); if difference < 0 { - difference += 256; + difference = difference.strict_add(256); borrow = 1; } else { borrow = 0; } let dst = out.len().strict_sub(index).strict_sub(1); - out[dst] = difference as u8; + out[dst] = difference.to_le_bytes()[0]; } if borrow != 0 { @@ -8844,7 +8825,6 @@ fn private_sub_unsigned_be_to_len( Ok(private_import_canonical_unsigned_be(out)) } -#[allow(clippy::indexing_slicing)] fn private_exponentiate_representative( modulus: &RsaPublicModulus, exponent: &[u8], @@ -8856,6 +8836,9 @@ fn private_exponentiate_representative( if input.len() != bytes || out.len() != bytes || exponent.len() != bytes { return Err(RsaPrivateOpError::InvalidLength); } + let r2 = modulus + .montgomery_r2() + .ok_or(RsaPrivateOpError::RepresentativeOutOfRange)?; let mut t = SecretLimbs::zeroed(limbs.strict_mul(2).strict_add(2)); let mut representative = SecretLimbs::zeroed(limbs); @@ -8876,17 +8859,11 @@ fn private_exponentiate_representative( private_mont_mul( base.as_mut_slice(), representative.as_slice(), - modulus.montgomery_r2(), - modulus, - t.as_mut_slice(), - ); - private_mont_mul( - acc.as_mut_slice(), - one.as_slice(), - modulus.montgomery_r2(), + r2, modulus, t.as_mut_slice(), ); + private_mont_mul(acc.as_mut_slice(), one.as_slice(), r2, modulus, t.as_mut_slice()); let mut table = private_fixed_window_table(base.as_slice(), acc.as_slice(), modulus, t.as_mut_slice()); let mut index = 0; @@ -8896,23 +8873,27 @@ fn private_exponentiate_representative( &table, exponent_byte >> 4, modulus, - t.as_mut_slice(), - acc.as_mut_slice(), - squared.as_mut_slice(), - multiplied.as_mut_slice(), - selected.as_mut_slice(), + RsaPrivateWindowWorkspace { + t: t.as_mut_slice(), + acc: acc.as_mut_slice(), + squared: squared.as_mut_slice(), + multiplied: multiplied.as_mut_slice(), + selected: selected.as_mut_slice(), + }, ); private_exponentiate_window( &table, exponent_byte & 0x0f, modulus, - t.as_mut_slice(), - acc.as_mut_slice(), - squared.as_mut_slice(), - multiplied.as_mut_slice(), - selected.as_mut_slice(), + RsaPrivateWindowWorkspace { + t: t.as_mut_slice(), + acc: acc.as_mut_slice(), + squared: squared.as_mut_slice(), + multiplied: multiplied.as_mut_slice(), + selected: selected.as_mut_slice(), + }, ); - index += 1; + index = index.strict_add(1); } ct::zeroize_words(table.as_mut_slice()); @@ -8921,7 +8902,6 @@ fn private_exponentiate_representative( Ok(()) } -#[allow(clippy::indexing_slicing)] fn private_exponentiate_representative_with_scratch( modulus: &RsaPublicModulus, exponent: &[u8], @@ -8934,6 +8914,9 @@ fn private_exponentiate_representative_with_scratch( if input.len() != bytes || out.len() != bytes || exponent.len() != bytes { return Err(RsaPrivateOpError::InvalidLength); } + let r2 = modulus + .montgomery_r2() + .ok_or(RsaPrivateOpError::RepresentativeOutOfRange)?; let RsaPrivateExponentWorkspace { t, representative, @@ -8954,8 +8937,8 @@ fn private_exponentiate_representative_with_scratch( one.fill(0); one[0] = 1; - private_mont_mul(base, representative, modulus.montgomery_r2(), modulus, t); - private_mont_mul(acc, one, modulus.montgomery_r2(), modulus, t); + private_mont_mul(base, representative, r2, modulus, t); + private_mont_mul(acc, one, r2, modulus, t); private_fixed_window_table_into(table, base, acc, modulus, t); let mut index = 0; @@ -8965,23 +8948,27 @@ fn private_exponentiate_representative_with_scratch( table, exponent_byte >> 4, modulus, - t, - acc, - squared, - multiplied, - selected, + RsaPrivateWindowWorkspace { + t: &mut *t, + acc: &mut *acc, + squared: &mut *squared, + multiplied: &mut *multiplied, + selected: &mut *selected, + }, ); private_exponentiate_window( table, exponent_byte & 0x0f, modulus, - t, - acc, - squared, - multiplied, - selected, + RsaPrivateWindowWorkspace { + t: &mut *t, + acc: &mut *acc, + squared: &mut *squared, + multiplied: &mut *multiplied, + selected: &mut *selected, + }, ); - index += 1; + index = index.strict_add(1); } private_mont_reduce(reduced, acc, modulus, t); @@ -8989,7 +8976,6 @@ fn private_exponentiate_representative_with_scratch( Ok(()) } -#[allow(clippy::indexing_slicing)] fn private_fixed_window_table( base: &[u64], one_montgomery: &[u64], @@ -9002,7 +8988,6 @@ fn private_fixed_window_table( table } -#[allow(clippy::indexing_slicing)] fn private_fixed_window_table_into( table: &mut [u64], base: &[u64], @@ -9025,17 +9010,27 @@ fn private_fixed_window_table_into( } } -#[allow(clippy::too_many_arguments)] +struct RsaPrivateWindowWorkspace<'a> { + t: &'a mut [u64], + acc: &'a mut [u64], + squared: &'a mut [u64], + multiplied: &'a mut [u64], + selected: &'a mut [u64], +} + fn private_exponentiate_window( table: &[u64], window: u8, modulus: &RsaPublicModulus, - t: &mut [u64], - acc: &mut [u64], - squared: &mut [u64], - multiplied: &mut [u64], - selected: &mut [u64], + workspace: RsaPrivateWindowWorkspace<'_>, ) { + let RsaPrivateWindowWorkspace { + t, + acc, + squared, + multiplied, + selected, + } = workspace; for _ in 0..4 { private_mont_mul(squared, acc, acc, modulus, t); acc.copy_from_slice(squared); @@ -9046,13 +9041,14 @@ fn private_exponentiate_window( acc.copy_from_slice(multiplied); } -#[allow(clippy::indexing_slicing)] #[inline(always)] fn private_select_window_power(out: &mut [u64], table: &[u64], window: u8) { let limbs = out.len(); debug_assert_eq!(table.len(), limbs.strict_mul(PRIVATE_FIXED_WINDOW_TABLE_ENTRIES)); out.fill(0); - for index in 0..PRIVATE_FIXED_WINDOW_TABLE_ENTRIES as u8 { + let entry_count = + u8::try_from(PRIVATE_FIXED_WINDOW_TABLE_ENTRIES).expect("private fixed-window table entry count must fit u8"); + for index in 0..entry_count { let start = usize::from(index).strict_mul(limbs); let entry = &table[start..start.strict_add(limbs)]; let mask = core::hint::black_box(private_choice_eq_mask_u8(window, index)); @@ -9069,6 +9065,7 @@ fn private_select_window_power(out: &mut [u64], table: &[u64], window: u8) { #[cfg(feature = "diag")] #[inline(always)] +/// Run the four-limb RSA private fixed-window selector for diagnostic harnesses. pub fn diag_rsa_private_select_window_power_4(table: &[u64; 64], window: u8) -> [u64; 4] { let mut out = [0u64; 4]; private_select_window_power(&mut out, table, window); @@ -9077,6 +9074,7 @@ pub fn diag_rsa_private_select_window_power_4(table: &[u64; 64], window: u8) -> #[cfg(feature = "diag")] #[inline(always)] +/// Evaluate the fixed-width RSA private-component checks used by diagnostic harnesses. pub fn diag_rsa_private_component_validation_32(component: &[u8; 32], upper_bound: &[u8; 32], other: &[u8; 32]) -> u8 { let canonical = u8::from(is_canonical_positive_unsigned_be(component)); let less_than_bound = u8::from(ct_unsigned_be_lt_public_shape(component, upper_bound)); @@ -9102,6 +9100,9 @@ fn mod_mul_representatives( if left.len() != bytes || right.len() != bytes || out.len() != bytes { return Err(RsaPrivateOpError::InvalidLength); } + let r2 = modulus + .montgomery_r2() + .ok_or(RsaPrivateOpError::RepresentativeOutOfRange)?; let mut t = SecretLimbs::zeroed(limbs.strict_mul(2).strict_add(2)); let mut left_limbs = SecretLimbs::zeroed(limbs); @@ -9120,14 +9121,14 @@ fn mod_mul_representatives( private_mont_mul( left_mont.as_mut_slice(), left_limbs.as_slice(), - modulus.montgomery_r2(), + r2, modulus, t.as_mut_slice(), ); private_mont_mul( right_mont.as_mut_slice(), right_limbs.as_slice(), - modulus.montgomery_r2(), + r2, modulus, t.as_mut_slice(), ); @@ -9160,6 +9161,9 @@ fn mod_mul_representatives_with_scratch( if left.len() != bytes || right.len() != bytes || out.len() != bytes { return Err(RsaPrivateOpError::InvalidLength); } + let r2 = modulus + .montgomery_r2() + .ok_or(RsaPrivateOpError::RepresentativeOutOfRange)?; scratch.ensure_limb_count(limbs)?; let t_len = limbs.strict_mul(2).strict_add(2); @@ -9205,8 +9209,8 @@ fn mod_mul_representatives_with_scratch( return Err(RsaPrivateOpError::RepresentativeOutOfRange); } - private_mont_mul(left_mont, left_limbs, modulus.montgomery_r2(), modulus, t); - private_mont_mul(right_mont, right_limbs, modulus.montgomery_r2(), modulus, t); + private_mont_mul(left_mont, left_limbs, r2, modulus, t); + private_mont_mul(right_mont, right_limbs, r2, modulus, t); private_mont_mul(product_mont, left_mont, right_mont, modulus, t); private_mont_reduce(product, product_mont, modulus, t); limbs_to_be(product, out); @@ -9229,6 +9233,14 @@ fn private_mont_reduce(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulu } } +#[inline(always)] +fn split_u128(value: u128) -> (u64, u64) { + ( + u64::try_from(value & u128::from(u64::MAX)).expect("masked low half of u128 must fit u64"), + u64::try_from(value >> u64::BITS).expect("shifted high half of u128 must fit u64"), + ) +} + #[cfg(feature = "getrandom")] fn private_import_product_unsigned_be(left: &[u8], right: &[u8]) -> Option { let left = SecretLimbs::from_be(left); @@ -9236,22 +9248,24 @@ fn private_import_product_unsigned_be(left: &[u8], right: &[u8]) -> Option> 64; + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *limb = acc_low; + carry = acc_high; } let index = left_index.strict_add(right.as_slice().len()); for limb in product.as_mut_slice().get_mut(index..)?.iter_mut() { - let acc = u128::from(*limb).strict_add(carry); - *limb = acc as u64; - carry = acc >> 64; + let acc = u128::from(*limb).strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *limb = acc_low; + carry = acc_high; } if carry != 0 { return None; @@ -9274,7 +9288,7 @@ fn private_import_product_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: & let mut product = SecretLimbs::zeroed(out_limb_count); for (left_index, &left_limb) in left.as_slice().iter().enumerate() { - let mut carry = 0u128; + let mut carry = 0u64; for (right_index, &right_limb) in right.as_slice().iter().enumerate() { let index = left_index.strict_add(right_index); let Some(limb) = product.as_mut_slice().get_mut(index) else { @@ -9282,9 +9296,10 @@ fn private_import_product_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: & }; let acc = u128::from(*limb) .strict_add(u128::from(left_limb).strict_mul(u128::from(right_limb))) - .strict_add(carry); - *limb = acc as u64; - carry = acc >> 64; + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *limb = acc_low; + carry = acc_high; } let index = left_index.strict_add(right.as_slice().len()); @@ -9292,9 +9307,10 @@ fn private_import_product_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: & return Err(RsaKeyError::InvalidModulus); }; for limb in carry_limbs { - let acc = u128::from(*limb).strict_add(carry); - *limb = acc as u64; - carry = acc >> 64; + let acc = u128::from(*limb).strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *limb = acc_low; + carry = acc_high; } if carry != 0 { return Err(RsaKeyError::InvalidModulus); @@ -9321,7 +9337,6 @@ fn private_import_product_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: & Ok(()) } -#[allow(clippy::indexing_slicing)] fn private_product_add_unsigned_be_to_fixed( left: &[u8], right: &[u8], @@ -9362,7 +9377,7 @@ fn private_product_add_unsigned_be_to_fixed( product.fill(0); for (left_index, &left_limb) in left_limbs.iter().enumerate() { - let mut carry = 0u128; + let mut carry = 0u64; for (right_index, &right_limb) in right_limbs.iter().enumerate() { let index = left_index.strict_add(right_index); let limb = product @@ -9370,9 +9385,10 @@ fn private_product_add_unsigned_be_to_fixed( .ok_or(RsaPrivateOpError::RepresentativeOutOfRange)?; let acc = u128::from(*limb) .strict_add(u128::from(left_limb).strict_mul(u128::from(right_limb))) - .strict_add(carry); - *limb = acc as u64; - carry = acc >> 64; + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *limb = acc_low; + carry = acc_high; } let index = left_index.strict_add(right_limbs.len()); @@ -9380,9 +9396,10 @@ fn private_product_add_unsigned_be_to_fixed( .get_mut(index..) .ok_or(RsaPrivateOpError::RepresentativeOutOfRange)?; for limb in carry_limbs { - let acc = u128::from(*limb).strict_add(carry); - *limb = acc as u64; - carry = acc >> 64; + let acc = u128::from(*limb).strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *limb = acc_low; + carry = acc_high; } if carry != 0 { return Err(RsaPrivateOpError::RepresentativeOutOfRange); @@ -9401,7 +9418,7 @@ fn private_product_add_unsigned_be_to_fixed( .copied() .unwrap_or(0); let sum = u16::from(out[dst]).strict_add(u16::from(add_byte)).strict_add(carry); - out[dst] = sum as u8; + out[dst] = sum.to_le_bytes()[0]; carry = sum >> 8; } @@ -9439,7 +9456,6 @@ fn private_import_decrement_unsigned_be_to_fixed(bytes: &[u8], out: &mut [u8]) - } } -#[allow(clippy::indexing_slicing)] #[cfg(feature = "getrandom")] fn private_sub_small_unsigned_be_to_fixed( bytes: &[u8], @@ -9527,7 +9543,6 @@ fn private_import_unsigned_be_mod_to_fixed( Ok(()) } -#[allow(clippy::indexing_slicing)] fn add_bit_mod_in_place(value: &mut [u64], modulus: &[u64], bit: u8) { debug_assert_eq!(value.len(), modulus.len()); @@ -9559,7 +9574,6 @@ fn private_import_canonical_unsigned_be(mut bytes: Vec) -> SecretBigEndianBu SecretBigEndianBuffer::new(canonical) } -#[allow(clippy::indexing_slicing)] fn cmp_limbs(a: &[u64], b: &[u64]) -> core::cmp::Ordering { debug_assert_eq!(a.len(), b.len()); for index in (0..a.len()).rev() { @@ -9582,7 +9596,6 @@ fn ct_limbs_lt(left: &[u64], right: &[u64]) -> bool { borrow == 1 } -#[allow(clippy::indexing_slicing)] fn limb_bit_len(limbs: &[u64]) -> usize { for index in (0..limbs.len()).rev() { let limb = limbs[index]; @@ -9631,7 +9644,6 @@ fn ct_nonzero_u64(value: u64) -> u64 { (value | value.wrapping_neg()) >> 63 } -#[allow(clippy::indexing_slicing)] fn sub_modulus_in_place(value: &mut [u64], modulus: &[u64]) -> u64 { debug_assert_eq!(value.len(), modulus.len()); let mut borrow = 0u64; @@ -9644,7 +9656,6 @@ fn sub_modulus_in_place(value: &mut [u64], modulus: &[u64]) -> u64 { borrow } -#[allow(clippy::indexing_slicing)] fn add_modulus_masked(value: &mut [u64], modulus: &[u64], choice: u64) { debug_assert_eq!(value.len(), modulus.len()); let mask = 0u64.wrapping_sub(choice & 1); @@ -9658,7 +9669,6 @@ fn add_modulus_masked(value: &mut [u64], modulus: &[u64], choice: u64) { } } -#[allow(clippy::indexing_slicing)] fn subtract_modulus_if_needed(value: &mut [u64], modulus: &[u64], extra: u64) { debug_assert_eq!(value.len(), modulus.len()); let borrow = sub_modulus_in_place(value, modulus); @@ -9667,7 +9677,6 @@ fn subtract_modulus_if_needed(value: &mut [u64], modulus: &[u64], extra: u64) { } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing)] fn add_mod_in_place(value: &mut [u64], addend: &[u64], modulus: &[u64]) { debug_assert_eq!(value.len(), addend.len()); debug_assert_eq!(value.len(), modulus.len()); @@ -9683,7 +9692,6 @@ fn add_mod_in_place(value: &mut [u64], addend: &[u64], modulus: &[u64]) { subtract_modulus_if_needed(value, modulus, carry); } -#[allow(clippy::indexing_slicing)] fn double_mod_in_place(value: &mut [u64], modulus: &[u64]) { debug_assert_eq!(value.len(), modulus.len()); let mut carry = 0u64; @@ -9715,7 +9723,6 @@ fn mul_mod_bitserial(out: &mut [u64], a: &[u64], b: &[u64], modulus: &[u64], add } } -#[allow(clippy::indexing_slicing)] fn pow2_mod_into(out: &mut [u64], bits: usize, modulus: &[u64]) { out.fill(0); let modulus_bits = limb_bit_len(modulus); @@ -9736,27 +9743,23 @@ fn montgomery_n0(n0: u64) -> u64 { inv.wrapping_neg() } -#[allow(clippy::indexing_slicing)] fn mont_square_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let _ = tmp; mont_square_product(value, modulus, t); } -#[allow(clippy::indexing_slicing)] fn mont_mul_in_place_left(left: &mut [u64], right: &[u64], tmp: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { copy_limbs(tmp, left); mont_mul(left, tmp, right, modulus, t); } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing)] fn mont_square_comba_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { copy_limbs(tmp, value); mont_mul_comba(value, tmp, tmp, modulus, t); } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing)] fn mont_mul_comba_in_place_left( left: &mut [u64], right: &[u64], @@ -9768,7 +9771,6 @@ fn mont_mul_comba_in_place_left( mont_mul_comba(left, tmp, right, modulus, t); } -#[allow(clippy::indexing_slicing)] fn mont_square_cios_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { #[cfg(all( target_arch = "aarch64", @@ -9816,7 +9818,6 @@ fn mont_square_cios_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPu mont_mul_cios(value, tmp, tmp, modulus, t); } -#[allow(clippy::indexing_slicing)] fn mont_mul_cios_in_place_left( left: &mut [u64], right: &[u64], @@ -9881,14 +9882,12 @@ fn mont_mul_cios_in_place_left( } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing)] fn mont_square_cios_portable_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { copy_limbs(tmp, value); mont_mul_cios_portable(value, tmp, tmp, modulus, t); } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing)] fn mont_mul_cios_portable_in_place_left( left: &mut [u64], right: &[u64], @@ -9913,7 +9912,6 @@ fn use_public_cios_montgomery(modulus: &RsaPublicModulus) -> bool { modulus.limbs.len() <= 128 } -#[allow(clippy::indexing_slicing)] #[cfg(feature = "diag")] fn mont_square_auto_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { if use_public_cios_montgomery(modulus) { @@ -9923,7 +9921,6 @@ fn mont_square_auto_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPu } } -#[allow(clippy::indexing_slicing)] #[cfg(feature = "diag")] fn mont_mul_auto_in_place_left( left: &mut [u64], @@ -9957,7 +9954,6 @@ fn mont_reduce_auto(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, } } -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_mul_cios(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10003,7 +9999,6 @@ fn mont_mul_cios(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModul mont_mul_cios_portable(out, a, b, modulus, t); } -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_mul_cios_portable(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10013,14 +10008,15 @@ fn mont_mul_cios_portable(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPu t[..n.strict_add(2)].fill(0); - for i in 0..n { + for &b_limb in &b[..n] { let mut carry = 0u64; - for j in 0..n { - let acc = u128::from(t[j]) - .strict_add(u128::from(a[j]).strict_mul(u128::from(b[i]))) + for (t_limb, &a_limb) in t[..n].iter_mut().zip(&a[..n]) { + let acc = u128::from(*t_limb) + .strict_add(u128::from(a_limb).strict_mul(u128::from(b_limb))) .strict_add(u128::from(carry)); - t[j] = acc as u64; - carry = (acc >> 64) as u64; + let (acc_low, acc_high) = split_u128(acc); + *t_limb = acc_low; + carry = acc_high; } let (sum, overflow) = t[n].overflowing_add(carry); t[n] = sum; @@ -10028,19 +10024,20 @@ fn mont_mul_cios_portable(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPu let q = t[0].wrapping_mul(modulus.n0); carry = 0; - for j in 0..n { - let acc = u128::from(t[j]) - .strict_add(u128::from(q).strict_mul(u128::from(modulus.limbs[j]))) + for (t_limb, &modulus_limb) in t[..n].iter_mut().zip(modulus.limbs.iter()) { + let acc = u128::from(*t_limb) + .strict_add(u128::from(q).strict_mul(u128::from(modulus_limb))) .strict_add(u128::from(carry)); - t[j] = acc as u64; - carry = (acc >> 64) as u64; + let (acc_low, acc_high) = split_u128(acc); + *t_limb = acc_low; + carry = acc_high; } let (sum, overflow) = t[n].overflowing_add(carry); t[n] = sum; t[n.strict_add(1)] = t[n.strict_add(1)].strict_add(u64::from(overflow)); - for j in 0..=n { - t[j] = t[j.strict_add(1)]; + for index in 0..=n { + t[index] = t[index.strict_add(1)]; } t[n.strict_add(1)] = 0; } @@ -10052,7 +10049,6 @@ fn mont_mul_cios_portable(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPu subtract_modulus_if_needed(out, &modulus.limbs, t[n]); } -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_reduce_cios(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10106,7 +10102,6 @@ fn mont_reduce_cios(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, mont_reduce_cios_portable(out, value, modulus, t); } -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_reduce_cios_portable(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10119,19 +10114,20 @@ fn mont_reduce_cios_portable(out: &mut [u64], value: &[u64], modulus: &RsaPublic for _ in 0..n { let q = t[0].wrapping_mul(modulus.n0); let mut carry = 0u64; - for j in 0..n { - let acc = u128::from(t[j]) - .strict_add(u128::from(q).strict_mul(u128::from(modulus.limbs[j]))) + for (t_limb, &modulus_limb) in t[..n].iter_mut().zip(modulus.limbs.iter()) { + let acc = u128::from(*t_limb) + .strict_add(u128::from(q).strict_mul(u128::from(modulus_limb))) .strict_add(u128::from(carry)); - t[j] = acc as u64; - carry = (acc >> 64) as u64; + let (acc_low, acc_high) = split_u128(acc); + *t_limb = acc_low; + carry = acc_high; } let (sum, overflow) = t[n].overflowing_add(carry); t[n] = sum; t[n.strict_add(1)] = t[n.strict_add(1)].strict_add(u64::from(overflow)); - for j in 0..=n { - t[j] = t[j.strict_add(1)]; + for index in 0..=n { + t[index] = t[index.strict_add(1)]; } t[n.strict_add(1)] = 0; } @@ -10144,7 +10140,6 @@ fn mont_reduce_cios_portable(out: &mut [u64], value: &[u64], modulus: &RsaPublic } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_mul_comba(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10156,17 +10151,18 @@ fn mont_mul_comba(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModu for i in 0..n { let q = t[i].wrapping_mul(modulus.n0); - let mut carry = 0u128; + let mut carry = 0u64; for j in 0..n { let index = i.strict_add(j); let acc = u128::from(q) .strict_mul(u128::from(modulus.limbs[j])) .strict_add(u128::from(t[index])) - .strict_add(carry); - t[index] = acc as u64; - carry = acc >> 64; + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + t[index] = acc_low; + carry = acc_high; } - add_carry(t, i.strict_add(n), carry as u64); + add_carry(t, i.strict_add(n), carry); } for (dst, src) in out.iter_mut().zip(t[n..n.strict_add(n)].iter().copied()) { @@ -10178,7 +10174,6 @@ fn mont_mul_comba(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModu } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing)] fn comba_mul_into(out: &mut [u64], a: &[u64], b: &[u64]) { debug_assert_eq!(a.len(), b.len()); let n = a.len(); @@ -10211,8 +10206,7 @@ fn comba_mul_into(out: &mut [u64], a: &[u64], b: &[u64]) { fn add_product_to_acc(acc_lo: &mut u64, acc_mid: &mut u64, acc_hi: &mut u64, a: u64, b: u64) { let product = u128::from(a).strict_mul(u128::from(b)); - let product_lo = product as u64; - let product_hi = (product >> 64) as u64; + let (product_lo, product_hi) = split_u128(product); let (lo, lo_overflow) = acc_lo.overflowing_add(product_lo); *acc_lo = lo; @@ -10222,7 +10216,6 @@ fn add_product_to_acc(acc_lo: &mut u64, acc_mid: &mut u64, acc_hi: &mut u64, a: *acc_hi = acc_hi.strict_add(u64::from(product_hi_overflow).strict_add(u64::from(lo_carry_overflow))); } -#[allow(clippy::indexing_slicing)] fn square_into_wide_product(out: &mut [u64], value: &[u64]) { let n = value.len(); debug_assert!(out.len() >= n.strict_mul(2).strict_add(2)); @@ -10261,7 +10254,6 @@ fn square_into_wide_product(out: &mut [u64], value: &[u64]) { out[product_limbs] = carry_hi; } -#[allow(clippy::indexing_slicing)] fn mont_square_product(out: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10271,14 +10263,18 @@ fn mont_square_product(out: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64 for i in 0..n { let q = t[i].wrapping_mul(modulus.n0); - let mut carry = 0u128; + let mut carry = 0u64; for j in 0..n { let index = i.strict_add(j); - let acc = u128::from(q) * u128::from(modulus.limbs[j]) + u128::from(t[index]) + carry; - t[index] = acc as u64; - carry = acc >> 64; + let acc = u128::from(q) + .strict_mul(u128::from(modulus.limbs[j])) + .strict_add(u128::from(t[index])) + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + t[index] = acc_low; + carry = acc_high; } - add_carry(t, i.strict_add(n), carry as u64); + add_carry(t, i.strict_add(n), carry); } for (dst, src) in out.iter_mut().zip(t[n..n.strict_add(n)].iter().copied()) { @@ -10289,7 +10285,6 @@ fn mont_square_product(out: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64 subtract_modulus_if_needed(out, &modulus.limbs, extra); } -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_mul(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10299,33 +10294,35 @@ fn mont_mul(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModulus, t t.fill(0); - for i in 0..n { - let mut carry = 0u128; - for j in 0..n { - let index = i.strict_add(j); - let acc = u128::from(a[j]) - .strict_mul(u128::from(b[i])) - .strict_add(u128::from(t[index])) - .strict_add(carry); - t[index] = acc as u64; - carry = acc >> 64; + for (i, &b_limb) in b[..n].iter().enumerate() { + let mut carry = 0u64; + let row_end = i.strict_add(n); + for (t_limb, &a_limb) in t[i..row_end].iter_mut().zip(&a[..n]) { + let acc = u128::from(a_limb) + .strict_mul(u128::from(b_limb)) + .strict_add(u128::from(*t_limb)) + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *t_limb = acc_low; + carry = acc_high; } - add_carry(t, i.strict_add(n), carry as u64); + add_carry(t, row_end, carry); } for i in 0..n { let q = t[i].wrapping_mul(modulus.n0); - let mut carry = 0u128; + let mut carry = 0u64; for j in 0..n { let index = i.strict_add(j); let acc = u128::from(q) .strict_mul(u128::from(modulus.limbs[j])) .strict_add(u128::from(t[index])) - .strict_add(carry); - t[index] = acc as u64; - carry = acc >> 64; + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + t[index] = acc_low; + carry = acc_high; } - add_carry(t, i.strict_add(n), carry as u64); + add_carry(t, i.strict_add(n), carry); } for (dst, src) in out.iter_mut().zip(t[n..n.strict_add(n)].iter().copied()) { @@ -10336,7 +10333,6 @@ fn mont_mul(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModulus, t subtract_modulus_if_needed(out, &modulus.limbs, extra); } -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_reduce(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10348,17 +10344,18 @@ fn mont_reduce(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, t: &m for i in 0..n { let q = t[i].wrapping_mul(modulus.n0); - let mut carry = 0u128; + let mut carry = 0u64; for j in 0..n { let index = i.strict_add(j); let acc = u128::from(q) .strict_mul(u128::from(modulus.limbs[j])) .strict_add(u128::from(t[index])) - .strict_add(carry); - t[index] = acc as u64; - carry = acc >> 64; + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + t[index] = acc_low; + carry = acc_high; } - add_carry(t, i.strict_add(n), carry as u64); + add_carry(t, i.strict_add(n), carry); } for (dst, src) in out.iter_mut().zip(t[n..n.strict_add(n)].iter().copied()) { @@ -10369,7 +10366,6 @@ fn mont_reduce(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, t: &m subtract_modulus_if_needed(out, &modulus.limbs, extra); } -#[allow(clippy::indexing_slicing)] fn add_carry(t: &mut [u64], index: usize, mut carry: u64) { for limb in &mut t[index..] { let (sum, overflow) = limb.overflowing_add(carry); @@ -10481,45 +10477,52 @@ mod tests { include_str!("../../testdata/rsa/nist_cavp/rsa_keygen_186_3_probable_prime_subset.json"); fn hex_to_vec(hex: &str) -> Vec { - assert_eq!(hex.len() % 2, 0); - let mut out = Vec::with_capacity(hex.len() / 2); - for chunk in hex.as_bytes().chunks_exact(2) { - out.push((hex_value(chunk[0]) << 4) | hex_value(chunk[1])); + let (chunks, remainder) = hex.as_bytes().as_chunks::<2>(); + assert!(remainder.is_empty(), "hex input must contain complete byte pairs"); + let mut out = Vec::with_capacity(chunks.len()); + for &[hi, lo] in chunks { + let hi = hex_value(hi).expect("hex fixtures must contain only ASCII hexadecimal digits"); + let lo = hex_value(lo).expect("hex fixtures must contain only ASCII hexadecimal digits"); + out.push((hi << 4) | lo); } out } - const fn hex_value(byte: u8) -> u8 { + const fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } #[cfg(feature = "getrandom")] fn json_field<'a>(value: &'a Value, name: &'static str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + value + .get(name) + .and_then(Value::as_str) + .expect("the CAVP fixture field must be a string") } #[cfg(feature = "getrandom")] fn test_keygen_drbg(label: &'static [u8]) -> RsaKeygenDrbg { - let mut seed = [0u8; RSA_KEYGEN_DRBG_ENTROPY_BYTES + RSA_KEYGEN_DRBG_NONCE_BYTES]; + let mut seed = [0u8; RSA_KEYGEN_DRBG_ENTROPY_BYTES.strict_add(RSA_KEYGEN_DRBG_NONCE_BYTES)]; for (index, byte) in seed.iter_mut().enumerate() { - *byte = (index as u8).wrapping_mul(17).wrapping_add(0xa5); + *byte = u8::try_from(index) + .expect("the RSA keygen DRBG fixture seed length must fit in one byte") + .wrapping_mul(17) + .wrapping_add(0xa5); } RsaKeygenDrbg::new(&seed, label) } fn rsa_miri_private_limb_widths() -> core::ops::RangeInclusive { - let min_factor_bits = RsaPublicKeyPolicy::LEGACY_VERIFICATION.min_modulus_bits.strict_add(1) / 2; - let max_factor_bits = RsaPublicKeyPolicy::LEGACY_VERIFICATION.max_modulus_bits.strict_add(1) / 2; + let min_factor_bits = RsaPublicKeyPolicy::LEGACY_VERIFICATION.min_modulus_bits.div_ceil(2); + let max_factor_bits = RsaPublicKeyPolicy::LEGACY_VERIFICATION.max_modulus_bits.div_ceil(2); let bits_per_limb = u64::BITS as usize; - let min_limbs = min_factor_bits.strict_add(bits_per_limb.strict_sub(1)) / bits_per_limb; - let max_limbs = max_factor_bits.strict_add(bits_per_limb.strict_sub(1)) / bits_per_limb; + let min_limbs = min_factor_bits.div_ceil(bits_per_limb); + let max_limbs = max_factor_bits.div_ceil(bits_per_limb); min_limbs..=max_limbs } @@ -10570,7 +10573,9 @@ mod tests { rsa_miri_fill_exponent_scratch(&mut scratch, MARKER); { - let workspace = scratch.workspace(limbs).unwrap(); + let workspace = scratch + .workspace(limbs) + .expect("valid RSA private-exponent scratch must bind to its limb width"); assert_eq!(workspace.t.len(), limbs.strict_mul(2).strict_add(2)); assert_eq!(workspace.representative.len(), limbs); assert_eq!(workspace.one.len(), limbs); @@ -10592,7 +10597,9 @@ mod tests { } } - for window in 0..PRIVATE_FIXED_WINDOW_TABLE_ENTRIES as u8 { + let window_count = u8::try_from(PRIVATE_FIXED_WINDOW_TABLE_ENTRIES) + .expect("the RSA fixed-window table entry count must fit in one byte"); + for window in 0..window_count { workspace.selected.fill(MARKER); private_select_window_power(workspace.selected, workspace.table, window); let start = usize::from(window).strict_mul(limbs); @@ -10657,7 +10664,9 @@ mod tests { fn rsa_miri_proof_miller_rabin_covers_small_prime_and_composite_paths() { let prime = 1009u16.to_be_bytes(); assert!(!has_small_prime_factor(&prime)); - assert!(private_import_is_probable_prime(&prime).unwrap()); + assert!( + private_import_is_probable_prime(&prime).expect("the small-prime RSA fixture must be accepted for testing") + ); let composite = [0x0f, 0x98, 0xa5]; assert_eq!( @@ -10665,7 +10674,10 @@ mod tests { 1009 * 1013 ); assert!(!has_small_prime_factor(&composite)); - assert!(!private_import_is_probable_prime(&composite).unwrap()); + assert!( + !private_import_is_probable_prime(&composite) + .expect("the composite RSA fixture must be evaluated without an arithmetic failure") + ); } #[test] @@ -10677,19 +10689,21 @@ mod tests { ([0x00], [0x00], [0x01], [0x00]), ] { let mut out = [0u8; 1]; - private_sub_mod_unsigned_be_to_fixed(&left, &right, &modulus, &mut out).unwrap(); + private_sub_mod_unsigned_be_to_fixed(&left, &right, &modulus, &mut out) + .expect("the bounded modular-subtraction fixture must be valid"); assert_eq!(out, expected); } let mut out = [0u8; 2]; - private_sub_mod_unsigned_be_to_fixed(&[0x00, 0x01], &[0x01, 0x00], &[0x01, 0x01], &mut out).unwrap(); + private_sub_mod_unsigned_be_to_fixed(&[0x00, 0x01], &[0x01, 0x00], &[0x01, 0x01], &mut out) + .expect("the borrow-propagation modular-subtraction fixture must be valid"); assert_eq!(out, [0x00, 0x02]); } fn integer_unsigned(value: &[u8]) -> Vec { let first_nonzero = value.iter().position(|&byte| byte != 0); let value = first_nonzero.map_or(&[0u8][..], |index| &value[index..]); - let mut encoded = Vec::with_capacity(value.len() + usize::from(value[0] & 0x80 != 0)); + let mut encoded = Vec::with_capacity(value.len().strict_add(usize::from(value[0] & 0x80 != 0))); if value[0] & 0x80 != 0 { encoded.push(0); } @@ -10747,7 +10761,11 @@ mod tests { params.extend_from_slice(&context_constructed(1, &x509_mgf1_algorithm(profile))); params.extend_from_slice(&context_constructed( 2, - &integer_unsigned(&u64::try_from(salt_len).unwrap().to_be_bytes()), + &integer_unsigned( + &u64::try_from(salt_len) + .expect("an RSA-PSS test salt length must fit in an unsigned DER integer") + .to_be_bytes(), + ), )); algorithm_identifier(ID_RSASSA_PSS_OID, Some(&tlv(TAG_SEQUENCE, ¶ms))) } @@ -10845,7 +10863,8 @@ ca5b455045218c7e196209c1c651702ece090a15e3cbcc265971300023a86fe9d34ad527e9ef03b7 }; RsaPrivateKey { - components: private_key_components_from_prevalidated_parts(&components, unsigned_be_bit_len(&modulus)).unwrap(), + components: private_key_components_from_prevalidated_parts(&components, unsigned_be_bit_len(&modulus)) + .expect("the prevalidated RSA private-key fixture must satisfy component invariants"), } } @@ -10985,18 +11004,20 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + der_len(value.len()).len() + value.len()); + let encoded_len = der_len(value.len()); + let capacity = 1usize.strict_add(encoded_len.len()).strict_add(value.len()); + let mut out = Vec::with_capacity(capacity); out.push(tag); - out.extend_from_slice(&der_len(value.len())); + out.extend_from_slice(&encoded_len); out.extend_from_slice(value); out } fn tlv_with_noncanonical_short_len(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(3 + value.len()); + let mut out = Vec::with_capacity(3usize.strict_add(value.len())); out.push(tag); out.push(0x81); - out.push(value.len() as u8); + out.push(u8::try_from(value.len()).expect("a noncanonical short DER fixture length must fit in one byte")); out.extend_from_slice(value); out } @@ -11009,7 +11030,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut out = Vec::with_capacity(der.len().strict_add(1)); out.push(tag); - out.push(0x80 | (len_len.strict_add(1) as u8)); + out.push(0x80 | u8::try_from(len_len.strict_add(1)).expect("a DER length-of-length must fit in one byte")); out.push(0); out.extend_from_slice(&der[2..]); out @@ -11017,14 +11038,17 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("a short DER length must fit in one byte")]; } let bytes = len.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .expect("a DER length of at least 128 must contain a nonzero byte"); let len_bytes = &bytes[first_nonzero..]; - let mut out = Vec::with_capacity(1 + len_bytes.len()); - out.push(0x80 | len_bytes.len() as u8); + let mut out = Vec::with_capacity(1usize.strict_add(len_bytes.len())); + out.push(0x80 | u8::try_from(len_bytes.len()).expect("a usize DER length prefix must fit in one byte")); out.extend_from_slice(len_bytes); out } @@ -11044,14 +11068,19 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 } fn pss_encoded_sha256(message: &[u8], em_bits: usize, salt_len: usize) -> Vec { - let em_len = em_bits.strict_add(7) / 8; + let em_len = em_bits.div_ceil(8); let h_len = Sha256::OUTPUT_SIZE; let db_len = em_len.strict_sub(h_len).strict_sub(1); let ps_len = db_len.strict_sub(salt_len).strict_sub(1); let mut salt = Vec::with_capacity(salt_len); for index in 0..salt_len { - salt.push((index as u8).wrapping_mul(17).wrapping_add(0xa5)); + salt.push( + u8::try_from(index) + .expect("the RSA-PSS test salt length must fit in one byte") + .wrapping_mul(17) + .wrapping_add(0xa5), + ); } let m_hash = Sha256::digest(message); @@ -11149,7 +11178,8 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn pkcs1_private_key_parser_preserves_components_and_public_key() { let der = test_pkcs1_private_key(); - let key = parse_pkcs1_private_key_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let key = parse_pkcs1_private_key_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("the canonical PKCS#1 private-key fixture must parse"); assert_eq!(prevalidated_test_private_key().to_pkcs1_der().as_bytes(), der); @@ -11263,7 +11293,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert_eq!(unsigned_be_bit_len(&prime_q), 1024); assert_eq!(prime_p, composite_p.to_bytes_be()); assert!(!has_small_prime_factor(&prime_p)); - assert!(!private_import_is_probable_prime(&prime_p).unwrap()); + assert!( + !private_import_is_probable_prime(&prime_p) + .expect("the balanced composite fixture must be evaluated without an arithmetic failure") + ); assert_eq!( RsaPrivateKey::from_components_with_policy( @@ -11340,6 +11373,20 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 fn private_key_components_debug_redacts_secret_material() { let key = prevalidated_test_private_key(); let debug = format!("{:?}", key.components); + let (blinding_factor, blinding_inverse) = factor_two_and_inverse(key.public_key().modulus()); + let blinding_debug = format!("{:?}", RsaBlindingPair::new(&blinding_factor, &blinding_inverse)); + let parts_debug = format!( + "{:?}", + rsa_private_key_parts( + &rsa_private_modulus(), + &rsa_private_exponent(), + &rsa_private_prime_p(), + &rsa_private_prime_q(), + &rsa_private_exponent_p(), + &rsa_private_exponent_q(), + &rsa_private_coefficient(), + ) + ); assert!(debug.contains("modulus_bits")); assert!(debug.contains("public_exponent")); @@ -11347,6 +11394,18 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert!(debug.contains("prime_p: \"****\"")); assert!(debug.contains("prime_q: \"****\"")); assert!(debug.contains("coefficient: \"****\"")); + assert!(parts_debug.contains("modulus_bits")); + assert!(parts_debug.contains("public_exponent: 65537")); + assert!(parts_debug.contains("private_exponent: \"****\"")); + assert!(parts_debug.contains("prime_p: \"****\"")); + assert!(parts_debug.contains("prime_q: \"****\"")); + assert!(parts_debug.contains("coefficient: \"****\"")); + assert_eq!( + blinding_debug, + "RsaBlindingPair { factor: \"****\", inverse: \"****\" }" + ); + assert!(!blinding_debug.contains(&format!("{blinding_factor:?}"))); + assert!(!blinding_debug.contains(&format!("{blinding_inverse:?}"))); for secret in [ rsa_private_exponent(), rsa_private_prime_p(), @@ -11356,6 +11415,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 rsa_private_coefficient(), ] { assert!(!debug.contains(&format!("{secret:?}"))); + assert!(!parts_debug.contains(&format!("{secret:?}"))); } assert_eq!( @@ -11382,15 +11442,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 .sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pkcs1v15_signature, ) - .unwrap(); + .expect("PKCS#1 v1.5 signing with a valid blinding pair must succeed"); key .public_key() .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &pkcs1v15_signature) - .unwrap(); + .expect("the generated PKCS#1 v1.5 signature must verify"); assert_eq!( key .public_key() @@ -11404,15 +11463,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaPssProfile::Sha256, message, &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pss_signature, ) - .unwrap(); + .expect("PSS signing with explicit salt and a valid blinding pair must succeed"); key .public_key() .verify_pss_with_salt_len(RsaPssProfile::Sha256, salt.len(), message, &pss_signature) - .unwrap(); + .expect("the generated PSS signature must verify with its explicit salt length"); assert_eq!( key .public_key() @@ -11426,7 +11484,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("OAEP encryption with a correctly sized deterministic seed must succeed"); let short_seed = [0x53; Sha256::OUTPUT_SIZE - 1]; ciphertext.fill(0xa5); assert_eq!( @@ -11443,26 +11501,24 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("OAEP fixture encryption must succeed after the rejected short seed"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep_with_blinding_factor( RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, ) - .unwrap(); + .expect("OAEP decryption with the matching label and blinding pair must succeed"); assert_eq!(&decrypted[..decrypted_len], plaintext); assert_eq!( key.decrypt_oaep_with_blinding_factor( RsaOaepProfile::Sha256, b"wrong label", &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, ), Err(RsaPrivateOpError::DecryptionFailed) @@ -11473,27 +11529,25 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_padding, &mut pkcs1v15_ciphertext) - .unwrap(); + .expect("PKCS#1 v1.5 encryption with a valid deterministic padding string must succeed"); let mut pkcs1v15_decrypted = vec![0u8; key.signature_len()]; let pkcs1v15_decrypted_len = key .decrypt_pkcs1v15_with_blinding_factor( &pkcs1v15_ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pkcs1v15_decrypted, ) - .unwrap(); + .expect("PKCS#1 v1.5 decryption with a valid blinding pair must succeed"); assert_eq!(&pkcs1v15_decrypted[..pkcs1v15_decrypted_len], pkcs1v15_plaintext); let mut scratch = key.private_scratch(); let pkcs1v15_decrypted_len = key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( &pkcs1v15_ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pkcs1v15_decrypted, &mut scratch, ) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 decryption with a valid blinding pair must succeed"); assert_eq!(&pkcs1v15_decrypted[..pkcs1v15_decrypted_len], pkcs1v15_plaintext); } @@ -11534,9 +11588,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 fn private_key_invalid_blinding_clears_signing_and_decryption_outputs() { let key = prevalidated_test_private_key(); let message = b"rscrypto RSA invalid blinding output clearing"; - let (blinding_factor, blinding_factor_inverse) = factor_two_and_inverse(key.public_key().modulus()); - let mut bad_blinding_inverse = blinding_factor_inverse.clone(); - *bad_blinding_inverse.last_mut().unwrap() ^= 1; + let (blinding_factor, mut bad_blinding_inverse) = factor_two_and_inverse(key.public_key().modulus()); + *bad_blinding_inverse + .last_mut() + .expect("the fixed-width blinding inverse must contain at least one byte") ^= 1; let mut scratch = key.private_scratch(); let mut pkcs1v15_signature = vec![0xa5; key.signature_len()]; @@ -11544,8 +11599,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key.sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut pkcs1v15_signature, ), Err(RsaPrivateOpError::InvalidBlindingFactor) @@ -11557,8 +11611,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key.sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut pkcs1v15_signature, &mut scratch, ), @@ -11573,8 +11626,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaPssProfile::Sha256, message, &salt, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut pss_signature, ), Err(RsaPrivateOpError::InvalidBlindingFactor) @@ -11587,8 +11639,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaPssProfile::Sha256, message, &salt, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut pss_signature, &mut scratch, ), @@ -11603,7 +11654,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("the OAEP invalid-blinding fixture must encrypt successfully"); let mut decrypted = vec![0xa5; key.signature_len()]; assert_eq!( @@ -11611,8 +11662,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut decrypted, ), Err(RsaPrivateOpError::InvalidBlindingFactor) @@ -11625,8 +11675,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut decrypted, &mut scratch, ), @@ -11640,14 +11689,13 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_seed, &mut pkcs1v15_ciphertext) - .unwrap(); + .expect("the PKCS#1 v1.5 invalid-blinding fixture must encrypt successfully"); let mut pkcs1v15_decrypted = vec![0xa5; key.signature_len()]; assert_eq!( key.decrypt_pkcs1v15_with_blinding_factor( &pkcs1v15_ciphertext, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut pkcs1v15_decrypted, ), Err(RsaPrivateOpError::InvalidBlindingFactor) @@ -11658,8 +11706,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert_eq!( key.decrypt_pkcs1v15_with_blinding_factor_and_scratch( &pkcs1v15_ciphertext, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut pkcs1v15_decrypted, &mut scratch, ), @@ -11677,7 +11724,8 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut valid = masked_oaep_sha256_from_decoded_db(&seed, &decoded_db); let mut out = vec![0u8; message.len()]; - let len = decode_oaep::(label, &mut valid, &mut out).unwrap(); + let len = decode_oaep::(label, &mut valid, &mut out) + .expect("the canonical OAEP encoded-message fixture must decode"); assert_eq!(len, message.len()); assert_eq!(out, message); @@ -11725,10 +11773,12 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let message = b"abc"; let seed = [0x7bu8; 10]; let mut encoded = [0u8; 16]; - encode_pkcs1v15_encryption_with_seed(message, &seed, &mut encoded).unwrap(); + encode_pkcs1v15_encryption_with_seed(message, &seed, &mut encoded) + .expect("the canonical PKCS#1 v1.5 encryption-padding fixture must encode"); let mut out = [0u8; 8]; - let len = decode_pkcs1v15_encryption(&encoded, &mut out).unwrap(); + let len = decode_pkcs1v15_encryption(&encoded, &mut out) + .expect("the canonical PKCS#1 v1.5 encryption-padding fixture must decode"); assert_eq!(&out[..len], message); let mut zero_seed = seed; @@ -11781,8 +11831,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, ) .unwrap(); @@ -11794,8 +11843,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, &mut scratch, ) @@ -11819,8 +11867,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, b"wrong label", &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, ), Err(RsaPrivateOpError::DecryptionFailed) @@ -11832,8 +11879,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, b"wrong label", &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, &mut scratch, ), @@ -11865,8 +11911,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, label, &tampered_tail, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, ), Err(RsaPrivateOpError::DecryptionFailed) @@ -11878,8 +11923,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, label, &tampered_tail, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, &mut scratch, ), @@ -11910,8 +11954,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, label, &zero_representative, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, ), Err(RsaPrivateOpError::DecryptionFailed) @@ -11923,8 +11966,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, label, &zero_representative, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, &mut scratch, ), @@ -11966,36 +12008,41 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("valid PKCS#1 v1.5 test plaintext and seed must encrypt"); let mut out = vec![0u8; key.signature_len()]; let decrypted_len = key - .decrypt_pkcs1v15_with_blinding_factor(&ciphertext, &blinding_factor, &blinding_factor_inverse, &mut out) - .unwrap(); + .decrypt_pkcs1v15_with_blinding_factor( + &ciphertext, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), + &mut out, + ) + .expect("valid PKCS#1 v1.5 ciphertext must decrypt with explicit blinding"); assert_eq!(&out[..decrypted_len], plaintext); let mut scratch = key.private_scratch(); let decrypted_len = key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, &mut scratch, ) - .unwrap(); + .expect("valid PKCS#1 v1.5 ciphertext must decrypt with caller scratch"); assert_eq!(&out[..decrypted_len], plaintext); #[cfg(feature = "getrandom")] { - let decrypted_len = key.decrypt_pkcs1v15(&ciphertext, &mut out).unwrap(); + let decrypted_len = key + .decrypt_pkcs1v15(&ciphertext, &mut out) + .expect("valid PKCS#1 v1.5 ciphertext must decrypt with random blinding"); assert_eq!(&out[..decrypted_len], plaintext); let decrypted_len = key .decrypt_pkcs1v15_with_scratch(&ciphertext, &mut out, &mut scratch) - .unwrap(); + .expect("valid PKCS#1 v1.5 ciphertext must decrypt with random blinding and caller scratch"); assert_eq!(&out[..decrypted_len], plaintext); } - let mut zero_seed = seed.clone(); + let mut zero_seed = seed; zero_seed[3] = 0; ciphertext.fill(0xa5); assert_eq!( @@ -12021,11 +12068,18 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 bad_block[2..10].fill(0xc5); bad_block[10] = 0; bad_block[11..11usize.strict_add(plaintext.len())].copy_from_slice(plaintext); - key.public_key().public_operation(&bad_block, &mut ciphertext).unwrap(); + key + .public_key() + .public_operation(&bad_block, &mut ciphertext) + .expect("malformed encoded block must remain a valid RSA representative"); let mut assert_decrypt_error = |ciphertext: &[u8]| { out.fill(0xa5); assert_eq!( - key.decrypt_pkcs1v15_with_blinding_factor(ciphertext, &blinding_factor, &blinding_factor_inverse, &mut out,), + key.decrypt_pkcs1v15_with_blinding_factor( + ciphertext, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), + &mut out, + ), Err(RsaPrivateOpError::DecryptionFailed) ); assert!(is_zero_unsigned_be(&out)); @@ -12033,8 +12087,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert_eq!( key.decrypt_pkcs1v15_with_blinding_factor_and_scratch( ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, &mut scratch, ), @@ -12059,24 +12112,34 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 }; bad_block[1] = 1; - key.public_key().public_operation(&bad_block, &mut ciphertext).unwrap(); + key + .public_key() + .public_operation(&bad_block, &mut ciphertext) + .expect("wrong block type must remain a valid RSA representative"); assert_decrypt_error(&ciphertext); bad_block[1] = 2; bad_block[5] = 0; - key.public_key().public_operation(&bad_block, &mut ciphertext).unwrap(); + key + .public_key() + .public_operation(&bad_block, &mut ciphertext) + .expect("short padding block must remain a valid RSA representative"); assert_decrypt_error(&ciphertext); bad_block[5] = 0xc5; bad_block[10..].fill(0xc5); - key.public_key().public_operation(&bad_block, &mut ciphertext).unwrap(); + key + .public_key() + .public_operation(&bad_block, &mut ciphertext) + .expect("missing separator block must remain a valid RSA representative"); assert_decrypt_error(&ciphertext); } #[test] fn key_der_exports_roundtrip_through_strict_importers() { let pkcs1 = test_pkcs1_private_key(); - let key = RsaPrivateKey::from_pkcs1_der_with_policy(&pkcs1, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let key = RsaPrivateKey::from_pkcs1_der_with_policy(&pkcs1, &RsaPublicKeyPolicy::legacy_verification()) + .expect("PKCS#1 fixture must import"); let rsa_algorithm = algorithm_identifier(RSA_ENCRYPTION_OID, Some(&null())); assert_eq!( @@ -12090,22 +12153,25 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let exported_pkcs1 = key.to_pkcs1_der(); let imported_pkcs1 = - RsaPrivateKey::from_pkcs1_der_with_policy(&exported_pkcs1, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + RsaPrivateKey::from_pkcs1_der_with_policy(&exported_pkcs1, &RsaPublicKeyPolicy::legacy_verification()) + .expect("exported PKCS#1 private key must re-import"); assert_eq!(imported_pkcs1.public_key(), key.public_key()); let exported_pkcs8 = key.to_pkcs8_der(); let imported_pkcs8 = - RsaPrivateKey::from_pkcs8_der_with_policy(&exported_pkcs8, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + RsaPrivateKey::from_pkcs8_der_with_policy(&exported_pkcs8, &RsaPublicKeyPolicy::legacy_verification()) + .expect("exported PKCS#8 private key must re-import"); assert_eq!(imported_pkcs8.public_key(), key.public_key()); let public_pkcs1 = key.public_key().to_pkcs1_der(); let imported_public_pkcs1 = - RsaPublicKey::from_pkcs1_der_with_policy(&public_pkcs1, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + RsaPublicKey::from_pkcs1_der_with_policy(&public_pkcs1, &RsaPublicKeyPolicy::legacy_verification()) + .expect("exported PKCS#1 public key must re-import"); assert_eq!(imported_public_pkcs1, *key.public_key()); let spki = key.public_key().to_spki_der(); - let imported_spki = - RsaPublicKey::from_spki_der_with_policy(&spki, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let imported_spki = RsaPublicKey::from_spki_der_with_policy(&spki, &RsaPublicKeyPolicy::legacy_verification()) + .expect("exported SPKI public key must re-import"); assert_eq!(imported_spki, *key.public_key()); } @@ -12120,7 +12186,8 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let coefficient = rsa_private_coefficient(); let policy = RsaPublicKeyPolicy::legacy_verification(); - let public = RsaPublicKey::from_modulus_exponent_with_policy(&modulus, 65_537, &policy).unwrap(); + let public = RsaPublicKey::from_modulus_exponent_with_policy(&modulus, 65_537, &policy) + .expect("valid raw public components must import"); assert_eq!(public.modulus(), modulus); assert_eq!(public.public_exponent().as_u64(), 65_537); @@ -12136,7 +12203,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ), &policy, ) - .unwrap(); + .expect("valid raw private components must import"); assert_eq!(key.public_key(), &public); let message = b"rscrypto raw RSA component import signing roundtrip"; @@ -12146,17 +12213,18 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 .sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, ) - .unwrap(); + .expect("imported raw private components must sign"); public .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &signature) - .unwrap(); + .expect("signature from imported raw components must verify"); let mut bad_coefficient = coefficient; - *bad_coefficient.last_mut().unwrap() ^= 1; + *bad_coefficient + .last_mut() + .expect("RSA coefficient fixture must be nonempty") ^= 1; assert_eq!( RsaPrivateKey::from_components_with_policy( rsa_private_key_parts( @@ -12285,7 +12353,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 Some(RsaKeyError::InvalidModulus) ); - let mut bad_coefficient = coefficient.clone(); + let mut bad_coefficient = coefficient; bad_coefficient.insert(0, 0); assert_eq!( RsaPrivateKey::from_components_with_policy( @@ -12315,11 +12383,11 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut pkcs1v15_signature = vec![0u8; key.signature_len()]; key .sign_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("PKCS#1 v1.5 signing with random blinding must succeed"); key .public_key() .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &pkcs1v15_signature) - .unwrap(); + .expect("PKCS#1 v1.5 signature must verify"); key .sign_pkcs1v15_with_scratch( RsaPkcs1v15Profile::Sha384, @@ -12327,35 +12395,35 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 &mut pkcs1v15_signature, &mut scratch, ) - .unwrap(); + .expect("PKCS#1 v1.5 signing with caller scratch must succeed"); key .public_key() .verify_pkcs1v15(RsaPkcs1v15Profile::Sha384, message, &pkcs1v15_signature) - .unwrap(); + .expect("caller-scratch PKCS#1 v1.5 signature must verify"); let mut pss_signature = vec![0u8; key.signature_len()]; key .sign_pss(RsaPssProfile::Sha256, message, &mut pss_signature) - .unwrap(); + .expect("PSS signing with random salt and blinding must succeed"); key .public_key() .verify_pss(RsaPssProfile::Sha256, message, &pss_signature) - .unwrap(); + .expect("PSS signature must verify"); key .sign_pss_with_scratch(RsaPssProfile::Sha384, message, &mut pss_signature, &mut scratch) - .unwrap(); + .expect("PSS signing with caller scratch must succeed"); key .public_key() .verify_pss(RsaPssProfile::Sha384, message, &pss_signature) - .unwrap(); + .expect("caller-scratch PSS signature must verify"); let pss_zero_salt = RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 0); key .sign_signature_with_scratch(pss_zero_salt, message, &mut pss_signature, &mut scratch) - .unwrap(); + .expect("zero-salt PSS signing with caller scratch must succeed"); key .public_key() .verify_signature(pss_zero_salt, message, &pss_signature) - .unwrap(); + .expect("zero-salt PSS signature must verify"); let label = b"rscrypto-getrandom-oaep"; let plaintext = b"normal oaep api"; @@ -12363,27 +12431,29 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .encrypt_oaep(RsaOaepProfile::Sha256, label, plaintext, &mut ciphertext) - .unwrap(); + .expect("OAEP encryption with random seed must succeed"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep(RsaOaepProfile::Sha256, label, &ciphertext, &mut decrypted) - .unwrap(); + .expect("OAEP ciphertext must decrypt with random blinding"); assert_eq!(&decrypted[..decrypted_len], plaintext); let decrypted_len = key .decrypt_oaep_with_scratch(RsaOaepProfile::Sha256, label, &ciphertext, &mut decrypted, &mut scratch) - .unwrap(); + .expect("OAEP ciphertext must decrypt with caller scratch"); assert_eq!(&decrypted[..decrypted_len], plaintext); let pkcs1v15_plaintext = b"normal pkcs1v15 encryption api"; key .public_key() .encrypt_pkcs1v15(pkcs1v15_plaintext, &mut ciphertext) - .unwrap(); - let decrypted_len = key.decrypt_pkcs1v15(&ciphertext, &mut decrypted).unwrap(); + .expect("PKCS#1 v1.5 encryption with random padding must succeed"); + let decrypted_len = key + .decrypt_pkcs1v15(&ciphertext, &mut decrypted) + .expect("PKCS#1 v1.5 ciphertext must decrypt with random blinding"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); let decrypted_len = key .decrypt_pkcs1v15_with_scratch(&ciphertext, &mut decrypted, &mut scratch) - .unwrap(); + .expect("PKCS#1 v1.5 ciphertext must decrypt with caller scratch"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); } @@ -12397,36 +12467,38 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut pkcs1v15_signature = vec![0u8; key.signature_len()]; key .sign_signature(pkcs1_profile, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("typed PKCS#1 v1.5 profile must sign"); key .public_key() .verify_signature(pkcs1_profile, message, &pkcs1v15_signature) - .unwrap(); + .expect("typed PKCS#1 v1.5 signature must verify"); let pss_zero_salt = RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 0); let mut pss_signature = vec![0u8; key.signature_len()]; - key.sign_signature(pss_zero_salt, message, &mut pss_signature).unwrap(); + key + .sign_signature(pss_zero_salt, message, &mut pss_signature) + .expect("typed zero-salt PSS profile must sign"); key .public_key() .verify_signature(pss_zero_salt, message, &pss_signature) - .unwrap(); + .expect("typed zero-salt PSS signature must verify"); key .sign_pss_with_salt_len(RsaPssProfile::Sha512, 24, message, &mut pss_signature) - .unwrap(); + .expect("explicit-salt-length PSS signing must succeed"); key .public_key() .verify_pss_with_salt_len(RsaPssProfile::Sha512, 24, message, &pss_signature) - .unwrap(); + .expect("explicit-salt-length PSS signature must verify"); let mut scratch = key.private_scratch(); key .sign_pss_with_salt_len_and_scratch(RsaPssProfile::Sha384, 16, message, &mut pss_signature, &mut scratch) - .unwrap(); + .expect("explicit-salt-length PSS signing with caller scratch must succeed"); key .public_key() .verify_pss_with_salt_len(RsaPssProfile::Sha384, 16, message, &pss_signature) - .unwrap(); + .expect("caller-scratch explicit-salt-length PSS signature must verify"); pss_signature.fill(0xa5); assert_eq!( @@ -12460,114 +12532,122 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let x509_pkcs1v15 = algorithm_identifier(SHA384_WITH_RSA_ENCRYPTION_OID, Some(&null())); key .sign_x509_signature_algorithm_der(&x509_pkcs1v15, message, &mut signature) - .unwrap(); - let x509_pkcs1v15_profile = RsaSignatureProfile::from_x509_signature_algorithm_der(&x509_pkcs1v15).unwrap(); + .expect("X.509 PKCS#1 v1.5 algorithm must sign"); + let x509_pkcs1v15_profile = RsaSignatureProfile::from_x509_signature_algorithm_der(&x509_pkcs1v15) + .expect("X.509 PKCS#1 v1.5 algorithm must parse"); key .public_key() .verify_signature(x509_pkcs1v15_profile, message, &signature) - .unwrap(); + .expect("X.509 PKCS#1 v1.5 signature must verify"); key .sign_x509_signature_algorithm_der_with_scratch(&x509_pkcs1v15, message, &mut signature, &mut scratch) - .unwrap(); + .expect("X.509 PKCS#1 v1.5 algorithm must sign with caller scratch"); key .public_key() .verify_signature(x509_pkcs1v15_profile, message, &signature) - .unwrap(); + .expect("caller-scratch X.509 PKCS#1 v1.5 signature must verify"); let x509_pss = x509_pss_algorithm(RsaPssProfile::Sha256, 20); key .sign_x509_signature_algorithm_der(&x509_pss, message, &mut signature) - .unwrap(); - let x509_pss_profile = RsaSignatureProfile::from_x509_signature_algorithm_der(&x509_pss).unwrap(); + .expect("X.509 PSS algorithm must sign"); + let x509_pss_profile = + RsaSignatureProfile::from_x509_signature_algorithm_der(&x509_pss).expect("X.509 PSS algorithm must parse"); key .public_key() .verify_signature(x509_pss_profile, message, &signature) - .unwrap(); + .expect("X.509 PSS signature must verify"); key .sign_x509_signature_algorithm_der_with_scratch(&x509_pss, message, &mut signature, &mut scratch) - .unwrap(); + .expect("X.509 PSS algorithm must sign with caller scratch"); key .public_key() .verify_signature(x509_pss_profile, message, &signature) - .unwrap(); + .expect("caller-scratch X.509 PSS signature must verify"); key .sign_tls13_signature_scheme(0x0804, message, &mut signature) - .unwrap(); + .expect("TLS 1.3 rsa_pss_rsae_sha256 scheme must sign"); key .public_key() .verify_signature( - RsaSignatureProfile::from_tls13_signature_scheme(0x0804).unwrap(), + RsaSignatureProfile::from_tls13_signature_scheme(0x0804) + .expect("TLS 1.3 rsa_pss_rsae_sha256 scheme must map to a profile"), message, &signature, ) - .unwrap(); + .expect("TLS 1.3 rsa_pss_rsae_sha256 signature must verify"); key .sign_tls13_signature_scheme_with_scratch(0x0804, message, &mut signature, &mut scratch) - .unwrap(); + .expect("TLS 1.3 scheme must sign with caller scratch"); key .public_key() .verify_signature( - RsaSignatureProfile::from_tls13_signature_scheme(0x0804).unwrap(), + RsaSignatureProfile::from_tls13_signature_scheme(0x0804) + .expect("TLS 1.3 rsa_pss_rsae_sha256 scheme must map to a profile"), message, &signature, ) - .unwrap(); + .expect("caller-scratch TLS 1.3 signature must verify"); key .sign_tls_certificate_signature_scheme(0x0501, message, &mut signature) - .unwrap(); + .expect("TLS certificate rsa_pkcs1_sha384 scheme must sign"); key .public_key() .verify_signature( - RsaSignatureProfile::from_tls_certificate_signature_scheme(0x0501).unwrap(), + RsaSignatureProfile::from_tls_certificate_signature_scheme(0x0501) + .expect("TLS certificate rsa_pkcs1_sha384 scheme must map to a profile"), message, &signature, ) - .unwrap(); + .expect("TLS certificate rsa_pkcs1_sha384 signature must verify"); key .sign_tls_certificate_signature_scheme_with_scratch(0x0501, message, &mut signature, &mut scratch) - .unwrap(); + .expect("TLS certificate scheme must sign with caller scratch"); key .public_key() .verify_signature( - RsaSignatureProfile::from_tls_certificate_signature_scheme(0x0501).unwrap(), + RsaSignatureProfile::from_tls_certificate_signature_scheme(0x0501) + .expect("TLS certificate rsa_pkcs1_sha384 scheme must map to a profile"), message, &signature, ) - .unwrap(); + .expect("caller-scratch TLS certificate signature must verify"); let jwt_algorithm = RsaJwtAlgorithm::Ps512; key .jwt_signer(jwt_algorithm) .try_sign_into(message, &mut signature) - .unwrap(); + .expect("JWT PS512 signer must sign"); key .public_key() .jwt_verifier(jwt_algorithm) .verify("PS512", message, &signature) - .unwrap(); + .expect("JWT PS512 signature must verify"); key .sign_signature_with_scratch(jwt_algorithm.signature_profile(), message, &mut signature, &mut scratch) - .unwrap(); + .expect("JWT PS512 profile must sign with caller scratch"); key .public_key() .jwt_verifier(jwt_algorithm) .verify("PS512", message, &signature) - .unwrap(); + .expect("caller-scratch JWT PS512 signature must verify"); - key.sign_cose_algorithm_id(-257, message, &mut signature).unwrap(); + key + .sign_cose_algorithm_id(-257, message, &mut signature) + .expect("COSE RS256 algorithm must sign"); key .public_key() .verify_cose_algorithm_id(-257, message, &signature) - .unwrap(); + .expect("COSE RS256 signature must verify"); key .sign_cose_algorithm_id_with_scratch(-257, message, &mut signature, &mut scratch) - .unwrap(); + .expect("COSE RS256 algorithm must sign with caller scratch"); key .public_key() .verify_cose_algorithm_id(-257, message, &signature) - .unwrap(); + .expect("caller-scratch COSE RS256 signature must verify"); signature.fill(0xa5); assert_eq!( @@ -12637,7 +12717,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn private_key_random_blinding_factor_has_valid_crt_inverse() { let key = prevalidated_test_private_key(); - let blinding = key.components.random_blinding_factor().unwrap(); + let blinding = key + .components + .random_blinding_factor() + .expect("test key must yield an invertible random blinding factor"); let mut check = vec![0u8; key.signature_len()]; mod_mul_representatives( @@ -12646,7 +12729,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 blinding.inverse(), &mut check, ) - .unwrap(); + .expect("blinding factor and inverse must multiply modulo n"); assert!(check[..check.len().strict_sub(1)].iter().all(|&byte| byte == 0)); assert_eq!(check.last().copied(), Some(1)); @@ -12657,7 +12740,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 fn private_key_blinding_inverse_rejects_non_invertible_factor() { let key = prevalidated_test_private_key(); let mut factor = vec![0u8; key.signature_len()]; - left_pad_be(&rsa_private_prime_p(), &mut factor).unwrap(); + left_pad_be(&rsa_private_prime_p(), &mut factor).expect("prime fixture must fit the RSA representative width"); let mut inverse = vec![0u8; key.signature_len()]; assert_eq!( @@ -12678,16 +12761,15 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 .sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pkcs1v15_signature, &mut scratch, ) - .unwrap(); + .expect("PKCS#1 v1.5 signing with explicit blinding and caller scratch must succeed"); key .public_key() .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &pkcs1v15_signature) - .unwrap(); + .expect("caller-scratch PKCS#1 v1.5 signature must verify"); let salt = [0x42; Sha384::OUTPUT_SIZE]; let mut pss_signature = vec![0u8; key.signature_len()]; @@ -12696,16 +12778,15 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaPssProfile::Sha384, message, &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pss_signature, &mut scratch, ) - .unwrap(); + .expect("PSS signing with explicit salt, blinding, and caller scratch must succeed"); key .public_key() .verify_pss(RsaPssProfile::Sha384, message, &pss_signature) - .unwrap(); + .expect("caller-scratch PSS signature must verify"); let label = b"rscrypto-private-scratch-oaep"; let plaintext = b"private scratch OAEP roundtrip"; @@ -12714,19 +12795,18 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("valid OAEP fixture inputs must encrypt"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep_with_blinding_factor_and_scratch( RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, &mut scratch, ) - .unwrap(); + .expect("OAEP ciphertext must decrypt with explicit blinding and caller scratch"); assert_eq!(&decrypted[..decrypted_len], plaintext); let pkcs1v15_plaintext = b"private scratch RSAES-PKCS1-v1_5 roundtrip"; @@ -12734,16 +12814,15 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_seed, &mut ciphertext) - .unwrap(); + .expect("valid PKCS#1 v1.5 fixture inputs must encrypt"); let decrypted_len = key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, &mut scratch, ) - .unwrap(); + .expect("PKCS#1 v1.5 ciphertext must decrypt with explicit blinding and caller scratch"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); let mut smaller_scratch = wrong_width_private_scratch(&key); @@ -12751,8 +12830,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key.sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pkcs1v15_signature, &mut smaller_scratch, ), @@ -12772,8 +12850,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key.sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, &mut scratch, ), @@ -12824,22 +12901,27 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn keygen_generated_prime_satisfies_fips_a1_3_prime_constraints() { let mut drbg = test_keygen_drbg(b"generated-prime-constraints"); - let prime = keygen_generate_prime(&mut drbg, 128, 256, None, 256usize.strict_mul(5)).unwrap(); + let prime = keygen_generate_prime(&mut drbg, 128, 256, None, 256usize.strict_mul(5)) + .expect("deterministic prime search must complete"); assert!(keygen_probable_prime_meets_fips_lower_bound(&prime, 128)); assert_eq!(prime.last().copied().unwrap_or_default() & 1, 1); assert!(!has_small_prime_factor(&prime)); assert!(!keygen_conflicts_with_public_exponent(&prime)); - assert!(keygen_is_probable_prime(&mut drbg, &prime).unwrap()); + assert!(keygen_is_probable_prime(&mut drbg, &prime).expect("generated prime must pass a valid primality test")); } #[cfg(feature = "getrandom")] #[test] fn keygen_lcm_private_exponent_contract_is_enforced() { - let p_minus_one = private_import_decrement_unsigned_be(&rsa_private_prime_p()).unwrap(); - let q_minus_one = private_import_decrement_unsigned_be(&rsa_private_prime_q()).unwrap(); - let lambda = keygen_lcm_unsigned_be(p_minus_one.as_slice(), q_minus_one.as_slice()).unwrap(); - let phi = private_import_product_unsigned_be(p_minus_one.as_slice(), q_minus_one.as_slice()).unwrap(); + let p_minus_one = + private_import_decrement_unsigned_be(&rsa_private_prime_p()).expect("fixture prime p must be greater than zero"); + let q_minus_one = + private_import_decrement_unsigned_be(&rsa_private_prime_q()).expect("fixture prime q must be greater than zero"); + let lambda = keygen_lcm_unsigned_be(p_minus_one.as_slice(), q_minus_one.as_slice()) + .expect("fixture prime totients must have an LCM"); + let phi = private_import_product_unsigned_be(p_minus_one.as_slice(), q_minus_one.as_slice()) + .expect("fixture prime totient product must fit the supported width"); assert_eq!( private_import_unsigned_be_mod(phi.as_slice(), lambda.as_slice()).as_slice(), @@ -12863,7 +12945,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 accepted = Some(candidate); } } - let candidate = accepted.unwrap(); + let candidate = accepted.expect("deterministic search must find a FIPS-shaped candidate"); assert!( keygen_candidate_has_fixed_shape(&candidate, 128), "accepted FIPS A.1.3 candidate must be odd and full-width" @@ -12903,16 +12985,19 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn keygen_miller_rabin_accepts_prime_and_rejects_composite_for_fixed_bases() { let prime = 1009u16.to_be_bytes(); - let prime_modulus = private_component_modulus(&prime).unwrap(); + let prime_modulus = private_component_modulus(&prime).expect("nonzero prime fixture must define a modulus"); assert!( - keygen_miller_rabin_accepts_base(&prime_modulus, &[0, 63], 4, &1008u16.to_be_bytes(), &[0, 11]).unwrap(), + keygen_miller_rabin_accepts_base(&prime_modulus, &[0, 63], 4, &1008u16.to_be_bytes(), &[0, 11]) + .expect("valid prime fixture must admit a Miller-Rabin round"), "1009 must pass a direct Miller-Rabin round for base 11" ); let composite = 341u16.to_be_bytes(); - let composite_modulus = private_component_modulus(&composite).unwrap(); + let composite_modulus = + private_component_modulus(&composite).expect("nonzero composite fixture must define a modulus"); assert!( - !keygen_miller_rabin_accepts_base(&composite_modulus, &[0, 85], 2, &340u16.to_be_bytes(), &[0, 2]).unwrap(), + !keygen_miller_rabin_accepts_base(&composite_modulus, &[0, 85], 2, &340u16.to_be_bytes(), &[0, 2]) + .expect("valid composite fixture must admit a Miller-Rabin round"), "341 must fail a direct Miller-Rabin round for base 2" ); } @@ -12927,15 +13012,15 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 rsa_private_prime_p(), rsa_private_prime_q(), ) - .unwrap() - .unwrap(); + .expect("fixture primes must derive without arithmetic failure") + .expect("fixture primes must produce the requested modulus width"); let key = RsaPrivateKey { components }; assert_eq!(key.public_key().modulus(), modulus); assert_eq!(key.public_key().public_exponent().as_u64(), RSA_KEYGEN_PUBLIC_EXPONENT); let exported = key.to_pkcs1_der(); - let imported = - RsaPrivateKey::from_pkcs1_der_with_policy(&exported, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let imported = RsaPrivateKey::from_pkcs1_der_with_policy(&exported, &RsaPublicKeyPolicy::legacy_verification()) + .expect("generated PKCS#1 private key must re-import"); assert_eq!(imported.public_key(), key.public_key()); let message = b"rscrypto generated RSA component signing roundtrip"; @@ -12945,15 +13030,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 .sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, ) - .unwrap(); + .expect("key derived from fixture primes must sign"); key .public_key() .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &signature) - .unwrap(); + .expect("signature from fixture-prime key must verify"); let label = b"rscrypto-keygen-oaep"; let plaintext = b"generated component oaep roundtrip"; @@ -12962,18 +13046,17 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("fixture-prime OAEP inputs must encrypt"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep_with_blinding_factor( RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, ) - .unwrap(); + .expect("fixture-prime OAEP ciphertext must decrypt"); assert_eq!(&decrypted[..decrypted_len], plaintext); let pkcs1v15_plaintext = b"generated component RSAES-PKCS1-v1_5 roundtrip"; @@ -12981,10 +13064,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_seed, &mut ciphertext) - .unwrap(); + .expect("fixture-prime PKCS#1 v1.5 inputs must encrypt"); let decrypted_len = key - .decrypt_pkcs1v15_with_blinding_factor(&ciphertext, &blinding_factor, &blinding_factor_inverse, &mut decrypted) - .unwrap(); + .decrypt_pkcs1v15_with_blinding_factor( + &ciphertext, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), + &mut decrypted, + ) + .expect("fixture-prime PKCS#1 v1.5 ciphertext must decrypt"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); } @@ -13003,7 +13090,8 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut covered = Vec::new(); for test in tests { - let modulus_bits = test["mod"].as_u64().expect("CAVP modulus size must be numeric") as usize; + let modulus_bits = usize::try_from(test["mod"].as_u64().expect("CAVP modulus size must be numeric")) + .expect("CAVP modulus size must fit usize"); let components = keygen_build_private_key_from_primes( modulus_bits, &policy, @@ -13016,8 +13104,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert_eq!(key.public_key().modulus_bits(), modulus_bits); assert_eq!(key.public_key().public_exponent().as_u64(), RSA_KEYGEN_PUBLIC_EXPONENT); let mut primality_drbg = test_keygen_drbg(b"cavp-primality"); - assert!(keygen_is_probable_prime(&mut primality_drbg, key.components.prime_p.as_bytes()).unwrap()); - assert!(keygen_is_probable_prime(&mut primality_drbg, key.components.prime_q.as_bytes()).unwrap()); + assert!( + keygen_is_probable_prime(&mut primality_drbg, key.components.prime_p.as_bytes()) + .expect("CAVP prime p must admit a primality test") + ); + assert!( + keygen_is_probable_prime(&mut primality_drbg, key.components.prime_q.as_bytes()) + .expect("CAVP prime q must admit a primality test") + ); let (blinding_factor, blinding_factor_inverse) = factor_two_and_inverse(key.public_key().modulus()); let message = b"rscrypto NIST CAVP keygen candidate private operation"; @@ -13027,15 +13121,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 .sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pkcs1v15_signature, ) - .unwrap(); + .expect("CAVP-derived key must sign PKCS#1 v1.5"); key .public_key() .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &pkcs1v15_signature) - .unwrap(); + .expect("CAVP-derived PKCS#1 v1.5 signature must verify"); let salt = [0xa5; Sha256::OUTPUT_SIZE]; let mut pss_signature = vec![0u8; key.signature_len()]; @@ -13044,15 +13137,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaPssProfile::Sha256, message, &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pss_signature, ) - .unwrap(); + .expect("CAVP-derived key must sign PSS with explicit salt and blinding"); key .public_key() .verify_pss(RsaPssProfile::Sha256, message, &pss_signature) - .unwrap(); + .expect("CAVP-derived PSS signature must verify"); let label = b"rscrypto-cavp-keygen-oaep"; let plaintext = b"NIST CAVP keygen candidate OAEP"; @@ -13061,18 +13153,17 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("CAVP-derived OAEP inputs must encrypt"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep_with_blinding_factor( RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, ) - .unwrap(); + .expect("CAVP-derived OAEP ciphertext must decrypt"); assert_eq!(&decrypted[..decrypted_len], plaintext); let pkcs1v15_plaintext = b"NIST CAVP keygen candidate RSAES-PKCS1-v1_5"; @@ -13080,10 +13171,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_seed, &mut ciphertext) - .unwrap(); + .expect("CAVP-derived PKCS#1 v1.5 inputs must encrypt"); let decrypted_len = key - .decrypt_pkcs1v15_with_blinding_factor(&ciphertext, &blinding_factor, &blinding_factor_inverse, &mut decrypted) - .unwrap(); + .decrypt_pkcs1v15_with_blinding_factor( + &ciphertext, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), + &mut decrypted, + ) + .expect("CAVP-derived PKCS#1 v1.5 ciphertext must decrypt"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); covered.push(modulus_bits); @@ -13095,16 +13190,17 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[cfg(feature = "getrandom")] #[test] fn keygen_generate_with_policy_produces_usable_private_key_end_to_end() { - let key = RsaPrivateKey::generate_with_policy(2048, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let key = RsaPrivateKey::generate_with_policy(2048, &RsaPublicKeyPolicy::legacy_verification()) + .expect("legacy test policy must generate a 2048-bit key"); assert_eq!(key.public_key().modulus_bits(), 2048); assert_eq!(key.public_key().public_exponent().as_u64(), RSA_KEYGEN_PUBLIC_EXPONENT); let pkcs1 = key.to_pkcs1_der(); let pkcs8 = key.to_pkcs8_der(); - let pkcs1_imported = - RsaPrivateKey::from_pkcs1_der_with_policy(&pkcs1, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); - let pkcs8_imported = - RsaPrivateKey::from_pkcs8_der_with_policy(&pkcs8, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let pkcs1_imported = RsaPrivateKey::from_pkcs1_der_with_policy(&pkcs1, &RsaPublicKeyPolicy::legacy_verification()) + .expect("generated PKCS#1 key must re-import"); + let pkcs8_imported = RsaPrivateKey::from_pkcs8_der_with_policy(&pkcs8, &RsaPublicKeyPolicy::legacy_verification()) + .expect("generated PKCS#8 key must re-import"); assert_eq!(pkcs1_imported.public_key(), key.public_key()); assert_eq!(pkcs8_imported.public_key(), key.public_key()); @@ -13114,29 +13210,31 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut pkcs1v15_signature = vec![0u8; key.signature_len()]; key .sign_pkcs1v15($pkcs1_profile, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("generated key must sign PKCS#1 v1.5"); key .public_key() .verify_pkcs1v15($pkcs1_profile, message, &pkcs1v15_signature) - .unwrap(); + .expect("generated-key PKCS#1 v1.5 signature must verify"); let mut pss_signature = vec![0u8; key.signature_len()]; - key.sign_pss($pss_profile, message, &mut pss_signature).unwrap(); + key + .sign_pss($pss_profile, message, &mut pss_signature) + .expect("generated key must sign PSS"); key .public_key() .verify_pss($pss_profile, message, &pss_signature) - .unwrap(); + .expect("generated-key PSS signature must verify"); let label = b"rscrypto-generated-key-oaep"; let mut ciphertext = vec![0u8; key.signature_len()]; key .public_key() .diag_encrypt_oaep_with_seed($oaep_profile, label, $plaintext, &$seed, &mut ciphertext) - .unwrap(); + .expect("generated-key OAEP inputs must encrypt"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep($oaep_profile, label, &ciphertext, &mut decrypted) - .unwrap(); + .expect("generated-key OAEP ciphertext must decrypt"); assert_eq!(&decrypted[..decrypted_len], $plaintext); let pkcs1v15_plaintext = b"generated key RSAES-PKCS1-v1_5"; @@ -13144,8 +13242,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_seed, &mut ciphertext) - .unwrap(); - let decrypted_len = key.decrypt_pkcs1v15(&ciphertext, &mut decrypted).unwrap(); + .expect("generated-key PKCS#1 v1.5 inputs must encrypt"); + let decrypted_len = key + .decrypt_pkcs1v15(&ciphertext, &mut decrypted) + .expect("generated-key PKCS#1 v1.5 ciphertext must decrypt"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); }}; } @@ -13176,18 +13276,22 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[cfg(feature = "getrandom")] #[test] fn keygen_generate_default_modern_key_produces_usable_private_key_end_to_end() { - let key = RsaPrivateKey::generate(3072).unwrap(); + let key = RsaPrivateKey::generate(3072).expect("default policy must generate a 3072-bit key"); assert_eq!(key.public_key().modulus_bits(), 3072); assert_eq!(key.public_key().public_exponent().as_u64(), RSA_KEYGEN_PUBLIC_EXPONENT); let pkcs1 = key.to_pkcs1_der(); let pkcs8 = key.to_pkcs8_der(); assert_eq!( - RsaPrivateKey::from_pkcs1_der(&pkcs1).unwrap().public_key(), + RsaPrivateKey::from_pkcs1_der(&pkcs1) + .expect("generated PKCS#1 key must re-import") + .public_key(), key.public_key() ); assert_eq!( - RsaPrivateKey::from_pkcs8_der(&pkcs8).unwrap().public_key(), + RsaPrivateKey::from_pkcs8_der(&pkcs8) + .expect("generated PKCS#8 key must re-import") + .public_key(), key.public_key() ); @@ -13198,20 +13302,20 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let pkcs1v15_profile = RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256); key .sign_signature_with_scratch(pkcs1v15_profile, message, &mut signature, &mut scratch) - .unwrap(); + .expect("generated modern key must sign PKCS#1 v1.5 with caller scratch"); key .public_key() .verify_signature(pkcs1v15_profile, message, &signature) - .unwrap(); + .expect("generated modern-key PKCS#1 v1.5 signature must verify"); let pss_profile = RsaSignatureProfile::pss(RsaPssProfile::Sha256); key .sign_signature_with_scratch(pss_profile, message, &mut signature, &mut scratch) - .unwrap(); + .expect("generated modern key must sign PSS with caller scratch"); key .public_key() .verify_signature(pss_profile, message, &signature) - .unwrap(); + .expect("generated modern-key PSS signature must verify"); let label = b"rscrypto-generated-modern-key-oaep"; let plaintext = b"modern generated key OAEP roundtrip"; @@ -13220,11 +13324,11 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("generated modern-key OAEP inputs must encrypt"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep_with_scratch(RsaOaepProfile::Sha256, label, &ciphertext, &mut decrypted, &mut scratch) - .unwrap(); + .expect("generated modern-key OAEP ciphertext must decrypt with caller scratch"); assert_eq!(&decrypted[..decrypted_len], plaintext); let pkcs1v15_plaintext = b"modern generated key RSAES-PKCS1-v1_5"; @@ -13232,10 +13336,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_seed, &mut ciphertext) - .unwrap(); + .expect("generated modern-key PKCS#1 v1.5 inputs must encrypt"); let decrypted_len = key .decrypt_pkcs1v15_with_scratch(&ciphertext, &mut decrypted, &mut scratch) - .unwrap(); + .expect("generated modern-key PKCS#1 v1.5 ciphertext must decrypt with caller scratch"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); } @@ -13278,7 +13382,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert!( keygen_build_private_key_from_primes(modulus_bits, &policy, rsa_private_prime_p(), rsa_private_prime_p()) - .unwrap() + .expect("duplicate prime fixtures must complete validation") .is_none() ); assert!( @@ -13288,7 +13392,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 rsa_private_prime_p(), rsa_private_prime_q() ) - .unwrap() + .expect("undersized modulus candidates must complete validation") .is_none() ); assert!( @@ -13298,7 +13402,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 rsa_private_prime_p(), rsa_private_prime_q() ) - .unwrap() + .expect("oversized modulus candidates must complete validation") .is_none() ); @@ -13306,7 +13410,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut carry = 2u16; for byte in close_prime_q.iter_mut().rev() { let sum = u16::from(*byte).strict_add(carry); - *byte = sum as u8; + *byte = sum.to_le_bytes()[0]; carry = sum >> 8; if carry == 0 { break; @@ -13319,7 +13423,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ); assert!( keygen_build_private_key_from_primes(modulus_bits, &policy, rsa_private_prime_p(), close_prime_q) - .unwrap() + .expect("close-prime candidates must complete validation") .is_none() ); @@ -13331,7 +13435,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 break; } let sum = u64::from(*byte).strict_add(delta & 0xff); - *byte = sum as u8; + *byte = sum.to_le_bytes()[0]; delta = (delta >> 8).strict_add(sum >> 8); } assert_eq!(delta, 0); @@ -13341,7 +13445,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ); assert!( keygen_build_private_key_from_primes(modulus_bits, &policy, public_exponent_conflict, rsa_private_prime_q()) - .unwrap() + .expect("public-exponent-conflicting candidates must complete validation") .is_none() ); } @@ -13350,20 +13454,21 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn keygen_random_prime_search_returns_probable_prime() { let mut drbg = test_keygen_drbg(b"random-prime-search"); - let prime = keygen_generate_prime(&mut drbg, 128, 256, None, 256usize.strict_mul(5)).unwrap(); + let prime = keygen_generate_prime(&mut drbg, 128, 256, None, 256usize.strict_mul(5)) + .expect("deterministic prime search must complete"); assert_eq!(unsigned_be_bit_len(&prime), 128); assert_eq!(prime.last().copied().unwrap_or_default() & 1, 1); assert!(!has_small_prime_factor(&prime)); assert!(!keygen_conflicts_with_public_exponent(&prime)); assert!(keygen_probable_prime_meets_fips_lower_bound(&prime, 128)); - assert!(keygen_is_probable_prime(&mut drbg, &prime).unwrap()); + assert!(keygen_is_probable_prime(&mut drbg, &prime).expect("generated prime must admit a primality test")); } #[test] fn pkcs1_private_key_parser_applies_modulus_policy_before_component_checks() { let mut bad_q = rsa_private_prime_q(); - *bad_q.last_mut().unwrap() ^= 0x02; + *bad_q.last_mut().expect("prime q fixture must be nonempty") ^= 0x02; let der = test_pkcs1_private_key_with_components( &[0x01, 0x00, 0x01], &rsa_private_exponent(), @@ -13386,7 +13491,8 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let pkcs1 = test_pkcs1_private_key(); let rsa_algorithm = algorithm_identifier(RSA_ENCRYPTION_OID, Some(&null())); let der = test_pkcs8_private_key(&pkcs1, &rsa_algorithm); - let key = parse_pkcs8_private_key_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let key = parse_pkcs8_private_key_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("valid PKCS#8 fixture must parse"); assert_eq!(key.public_key().modulus(), rsa_private_modulus()); @@ -13495,7 +13601,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ); let mut bad_q = rsa_private_prime_q(); - let last = bad_q.last_mut().unwrap(); + let last = bad_q.last_mut().expect("prime q fixture must be nonempty"); *last ^= 0x02; let mut inconsistent_body = Vec::new(); for field in [ @@ -13521,7 +13627,9 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn pkcs1_private_key_parser_rejects_crt_congruence_mismatches() { let mut bad_exponent_p = rsa_private_exponent_p(); - *bad_exponent_p.last_mut().unwrap() ^= 0x02; + *bad_exponent_p + .last_mut() + .expect("CRT exponent p fixture must be nonempty") ^= 0x02; let bad_dp = test_pkcs1_private_key_with_crt( &[0x01, 0x00, 0x01], &bad_exponent_p, @@ -13534,7 +13642,9 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ); let mut bad_exponent_q = rsa_private_exponent_q(); - *bad_exponent_q.last_mut().unwrap() ^= 0x02; + *bad_exponent_q + .last_mut() + .expect("CRT exponent q fixture must be nonempty") ^= 0x02; let bad_dq = test_pkcs1_private_key_with_crt( &[0x01, 0x00, 0x01], &rsa_private_exponent_p(), @@ -13547,7 +13657,9 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ); let mut bad_coefficient = rsa_private_coefficient(); - *bad_coefficient.last_mut().unwrap() ^= 0x02; + *bad_coefficient + .last_mut() + .expect("CRT coefficient fixture must be nonempty") ^= 0x02; let bad_qinv = test_pkcs1_private_key_with_crt( &[0x01, 0x00, 0x01], &rsa_private_exponent_p(), @@ -13923,11 +14035,11 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut bytes = vec![0u8; words.strict_mul(8)]; for (i, byte) in bytes.iter_mut().enumerate() { *byte = 0xa5u8 - .wrapping_add((i as u8).wrapping_mul(0x3d)) - .wrapping_add((words as u8).wrapping_mul(0x17)); + .wrapping_add(i.to_le_bytes()[0].wrapping_mul(0x3d)) + .wrapping_add(words.to_le_bytes()[0].wrapping_mul(0x17)); } bytes[0] = 0xff; - *bytes.last_mut().unwrap() |= 1; + *bytes.last_mut().expect("RSA Montgomery test width must be nonzero") |= 1; RsaPublicModulus::new(&bytes, words.strict_mul(64)) } @@ -13948,7 +14060,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 state ^= state << 17; *limb = state; } - *limbs.last_mut().unwrap() &= 0x7fff_ffff_ffff_ffff; + *limbs.last_mut().expect("RSA Montgomery test width must be nonzero") &= 0x7fff_ffff_ffff_ffff; limbs[0] |= 1; limbs } @@ -14057,7 +14169,6 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 &mut asm_acc, &modulus.limbs, modulus.n0, - words, &mut asm_t, ); public_e65537_cios_portable(&mut portable_out, &a, &r2, &mut portable_acc, &modulus, &mut portable_t); @@ -14243,9 +14354,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn diag_spki_public_key_validation_matches_import_metadata() { let spki = include_bytes!("../../benches/rsa_fixtures/rsa3072_spki.der"); - let key = RsaPublicKey::from_spki_der(spki).unwrap(); + let key = RsaPublicKey::from_spki_der(spki).expect("RSA-3072 SPKI fixture must import"); let (modulus_len, modulus_bits, exponent) = - diag_rsa_validate_spki_public_key_der(spki, &RsaPublicKeyPolicy::default()).unwrap(); + diag_rsa_validate_spki_public_key_der(spki, &RsaPublicKeyPolicy::default()) + .expect("RSA-3072 SPKI fixture must pass diagnostic validation"); assert_eq!(modulus_len, key.modulus().len()); assert_eq!(modulus_bits, key.modulus_bits()); @@ -14256,7 +14368,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn diag_montgomery_r2_precompute_matches_imported_key() { let spki = include_bytes!("../../benches/rsa_fixtures/rsa3072_spki.der"); - let key = RsaPublicKey::from_spki_der(spki).unwrap(); + let key = RsaPublicKey::from_spki_der(spki).expect("RSA-3072 SPKI fixture must import"); let scratch = key.public_scratch(); let (_, scratch_r2) = scratch.montgomery.split_at(scratch.limb_count); @@ -14275,7 +14387,8 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let policy = RsaPublicKeyPolicy::legacy_verification().allow_legacy_small_exponents(); for exponent in [&[0x03][..], &[0x11][..], &[0x01, 0x00, 0x01][..]] { - let key = RsaPublicKey::from_pkcs1_der_with_policy(&test_pkcs1_public_key(&modulus, exponent), &policy).unwrap(); + let key = RsaPublicKey::from_pkcs1_der_with_policy(&test_pkcs1_public_key(&modulus, exponent), &policy) + .expect("test exponent must produce a valid public key"); let mut specialized = vec![0u8; key.modulus().len()]; let mut generic = vec![0u8; key.modulus().len()]; let mut specialized_scratch = key.public_scratch(); @@ -14283,8 +14396,9 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_operation_with_scratch(&input, &mut specialized, &mut specialized_scratch) - .unwrap(); - diag_rsa_public_operation_generic_exponent(&key, &input, &mut generic, &mut generic_scratch).unwrap(); + .expect("specialized public operation must accept the test representative"); + diag_rsa_public_operation_generic_exponent(&key, &input, &mut generic, &mut generic_scratch) + .expect("generic public operation must accept the test representative"); assert_eq!( generic, specialized, @@ -14302,7 +14416,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 &test_pkcs1_public_key(&modulus, &exponent), &RsaPublicKeyPolicy::legacy_verification(), ) - .unwrap(); + .expect("diagnostic backend test key must import"); let representative = key.modulus().to_vec(); let mut out = vec![0xa5; key.modulus().len()]; @@ -14416,13 +14530,13 @@ ec34e8c72cc58fd5324fbe1ddd9714909caedfaa38706cfa66d9bc1026ba3ec1188092392a54a\ &test_pkcs1_public_key(&modulus, &[0x01, 0x00, 0x01]), &RsaPublicKeyPolicy::legacy_verification(), ) - .unwrap(); + .expect("independent-vector public key must import"); let mut out = vec![0u8; key.modulus().len()]; let mut scratch = key.public_scratch(); key .public_operation_with_scratch(&input, &mut out, &mut scratch) - .unwrap(); + .expect("independent-vector representative must exponentiate"); assert_eq!(out, expected); } @@ -14506,10 +14620,12 @@ ec34e8c72cc58fd5324fbe1ddd9714909caedfaa38706cfa66d9bc1026ba3ec1188092392a54a\ &x509_certificate_fixture_public_key(), &RsaPublicKeyPolicy::legacy_verification(), ) - .unwrap(); + .expect("X.509 issuer fixture must import"); let certificate = x509_pkcs1v15_certificate_fixture(); - assert!(issuer.verify_x509_certificate_signature_der(&certificate).is_ok()); + issuer + .verify_x509_certificate_signature_der(&certificate) + .expect("fixture certificate signature must verify"); let mut tampered = certificate; if let Some(last) = tampered.last_mut() { @@ -14577,9 +14693,15 @@ ec34e8c72cc58fd5324fbe1ddd9714909caedfaa38706cfa66d9bc1026ba3ec1188092392a54a\ left in any::(), right in any::(), ) { - let left = left % modulus; - let right = right % modulus; - let expected = ((u64::from(left) + u64::from(modulus) - u64::from(right)) % u64::from(modulus)) as u32; + let left = left.rem_euclid(modulus); + let right = right.rem_euclid(modulus); + let expected = u32::try_from( + u64::from(left) + .strict_add(u64::from(modulus)) + .strict_sub(u64::from(right)) + .rem_euclid(u64::from(modulus)), + ) + .expect("reduced u32 modular difference must fit u32"); let mut out = [0u8; 4]; private_sub_mod_unsigned_be_to_fixed( @@ -14588,7 +14710,7 @@ ec34e8c72cc58fd5324fbe1ddd9714909caedfaa38706cfa66d9bc1026ba3ec1188092392a54a\ &modulus.to_be_bytes(), &mut out, ) - .unwrap(); + .expect("valid u32 modular subtraction inputs must succeed"); prop_assert_eq!(out, expected.to_be_bytes()); } @@ -14598,7 +14720,8 @@ ec34e8c72cc58fd5324fbe1ddd9714909caedfaa38706cfa66d9bc1026ba3ec1188092392a54a\ modulus[0] |= 0x80; modulus[255] |= 0x01; let exponent = RsaPublicExponent(65_537); - let bits = validate_modulus(&modulus, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let bits = validate_modulus(&modulus, &RsaPublicKeyPolicy::legacy_verification()) + .expect("generated full-width odd modulus must satisfy the legacy test policy"); let key = RsaPublicKey { modulus: RsaPublicModulus::new(&modulus, bits), exponent, diff --git a/src/auth/rsa_aarch64_asm.rs b/src/auth/rsa_aarch64_asm.rs index 2a158ced..c8d02b7e 100644 --- a/src/auth/rsa_aarch64_asm.rs +++ b/src/auth/rsa_aarch64_asm.rs @@ -3,8 +3,6 @@ //! This module owns the ABI boundary. `rsa.rs` owns all RSA validation, //! representative range checks, dispatch, and fallback semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; global_asm!(include_str!("asm/rscrypto_rsa_aarch64_apple_darwin.s")); @@ -74,7 +72,14 @@ pub(super) fn mont_mul_cios_words( // read/write ranges. The assembly does not retain pointers after returning. // 4. `out` does not alias `a`, `b`, or `modulus` in current non-in-place callers. unsafe { - rscrypto_rsa_bn_mul_mont_words_apple(out.as_mut_ptr(), a.as_ptr(), b.as_ptr(), modulus.as_ptr(), &n0, words) + rscrypto_rsa_bn_mul_mont_words_apple( + out.as_mut_ptr(), + a.as_ptr(), + b.as_ptr(), + modulus.as_ptr(), + &raw const n0, + words, + ) }; } @@ -104,7 +109,7 @@ pub(super) fn mont_square_cios_words_in_place( value.as_ptr(), value.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ) }; @@ -139,7 +144,7 @@ pub(super) fn mont_mul_cios_words_in_place_left( left.as_ptr(), right.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ) }; @@ -162,8 +167,8 @@ pub(super) fn mont_reduce_cios_words( // SAFETY: RSA Montgomery REDC assembly call because: // 1. This module is compiled only for macOS AArch64 and embeds the matching Darwin symbol. - // 2. The caller checks `out`, `value`, and `modulus` are 48 or 64 `u64` limbs and `t` has `2 * - // words + 2` scratch limbs. + // 2. The caller checks `out`, `value`, and `modulus` are 48, 64, or 128 `u64` limbs and `t` has + // `2 * words + 2` scratch limbs. // 3. The assembly writes only caller-provided scratch and performs final conditional subtraction // with mask selection, not data-dependent branches. // 4. All pointers are derived from live Rust slices and the assembly does not retain them. @@ -180,7 +185,6 @@ pub(super) fn mont_reduce_cios_words( } #[inline] -#[allow(clippy::too_many_arguments)] // Assembly ABI boundary; a wrapper struct would only repack these operands. pub(super) fn public_e65537_mont_words( out: &mut [u64], input: &[u64], @@ -188,9 +192,9 @@ pub(super) fn public_e65537_mont_words( acc: &mut [u64], modulus: &[u64], n0: u64, - words: usize, t: &mut [u64], ) { + let words = out.len(); debug_assert!(supports_bignum_mont_words(words)); debug_assert_eq!(out.len(), words); debug_assert_eq!(input.len(), words); @@ -212,7 +216,7 @@ pub(super) fn public_e65537_mont_words( input.as_ptr(), r2.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ); acc.copy_from_slice(out); @@ -222,7 +226,7 @@ pub(super) fn public_e65537_mont_words( acc.as_ptr(), acc.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ); } @@ -231,7 +235,7 @@ pub(super) fn public_e65537_mont_words( acc.as_ptr(), out.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ); diff --git a/src/auth/rsa_aarch64_linux_asm.rs b/src/auth/rsa_aarch64_linux_asm.rs index f32c6180..73872ed8 100644 --- a/src/auth/rsa_aarch64_linux_asm.rs +++ b/src/auth/rsa_aarch64_linux_asm.rs @@ -3,8 +3,6 @@ //! This module owns the ABI boundary. `rsa.rs` owns all RSA validation, //! representative range checks, dispatch, and fallback semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; global_asm!(include_str!("asm/rscrypto_rsa_aarch64_linux_elf.s")); @@ -74,7 +72,14 @@ pub(super) fn mont_mul_cios_words( // read/write ranges. The assembly does not retain pointers after returning. // 4. `out` does not alias `a`, `b`, or `modulus` in current non-in-place callers. unsafe { - rscrypto_rsa_bn_mul_mont_words_aarch64_elf(out.as_mut_ptr(), a.as_ptr(), b.as_ptr(), modulus.as_ptr(), &n0, words) + rscrypto_rsa_bn_mul_mont_words_aarch64_elf( + out.as_mut_ptr(), + a.as_ptr(), + b.as_ptr(), + modulus.as_ptr(), + &raw const n0, + words, + ) }; } @@ -104,7 +109,7 @@ pub(super) fn mont_square_cios_words_in_place( value.as_ptr(), value.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ) }; @@ -139,7 +144,7 @@ pub(super) fn mont_mul_cios_words_in_place_left( left.as_ptr(), right.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ) }; @@ -162,8 +167,8 @@ pub(super) fn mont_reduce_cios_words( // SAFETY: RSA Montgomery REDC assembly call because: // 1. This module is compiled only for Linux AArch64 and embeds the matching ELF symbol. - // 2. The caller checks `out`, `value`, and `modulus` are 48 or 64 `u64` limbs and `t` has `2 * - // words + 2` scratch limbs. + // 2. The caller checks `out`, `value`, and `modulus` are 48, 64, or 128 `u64` limbs and `t` has + // `2 * words + 2` scratch limbs. // 3. The assembly writes only caller-provided scratch and performs final conditional subtraction // with mask selection, not data-dependent branches. // 4. All pointers are derived from live Rust slices and the assembly does not retain them. @@ -180,7 +185,6 @@ pub(super) fn mont_reduce_cios_words( } #[inline] -#[allow(clippy::too_many_arguments)] // Assembly ABI boundary; a wrapper struct would only repack these operands. pub(super) fn public_e65537_mont_words( out: &mut [u64], input: &[u64], @@ -188,9 +192,9 @@ pub(super) fn public_e65537_mont_words( acc: &mut [u64], modulus: &[u64], n0: u64, - words: usize, t: &mut [u64], ) { + let words = out.len(); debug_assert!(supports_bignum_mont_words(words)); debug_assert_eq!(out.len(), words); debug_assert_eq!(input.len(), words); @@ -212,7 +216,7 @@ pub(super) fn public_e65537_mont_words( input.as_ptr(), r2.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ); acc.copy_from_slice(out); @@ -222,7 +226,7 @@ pub(super) fn public_e65537_mont_words( acc.as_ptr(), acc.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ); } @@ -231,7 +235,7 @@ pub(super) fn public_e65537_mont_words( acc.as_ptr(), out.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ); diff --git a/src/auth/rsa_x86_64_asm.rs b/src/auth/rsa_x86_64_asm.rs index fdb81119..224e2e9a 100644 --- a/src/auth/rsa_x86_64_asm.rs +++ b/src/auth/rsa_x86_64_asm.rs @@ -3,8 +3,6 @@ //! This module owns the ABI boundary. `rsa.rs` owns all RSA validation, //! representative range checks, dispatch, and fallback semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; use crate::platform::{caps, caps::x86}; @@ -73,7 +71,14 @@ pub(super) fn mont_mul_cios_words( // 4. All pointers are derived from live Rust slices and are valid for the assembly's fixed // read/write ranges. The assembly does not retain pointers after returning. let _ = unsafe { - rscrypto_rsa_bn_mulx4x_mont_x86_64_elf(out.as_mut_ptr(), a.as_ptr(), b.as_ptr(), modulus.as_ptr(), &n0, words) + rscrypto_rsa_bn_mulx4x_mont_x86_64_elf( + out.as_mut_ptr(), + a.as_ptr(), + b.as_ptr(), + modulus.as_ptr(), + &raw const n0, + words, + ) }; } @@ -92,7 +97,7 @@ pub(super) fn mont_square_cios_words_in_place( // SAFETY: RSA in-place Montgomery square ADX/BMI2 assembly call because: // 1. This module is compiled only for Linux x86-64 and embeds the matching ELF symbol. - // 2. `supports_bignum_mont_words` checks BMI2 and ADX before this call, matching the kernel's + // 2. `supports_bignum_mont_square_words` checks BMI2 and ADX before this call, matching the kernel's // `mulx/adcx/adox` instruction requirements. // 3. The caller checks `value` and `modulus` are supported `u64` limb widths satisfying the square // kernel's `num >= 8 && num % 8 == 0` precondition. @@ -101,7 +106,14 @@ pub(super) fn mont_square_cios_words_in_place( // `value` as both `out` and `a` preserves the in-place helper contract. // 5. All pointers are derived from live Rust slices and the assembly does not retain them. let _ = unsafe { - rscrypto_rsa_bn_sqr8x_mont_x86_64_elf(value.as_mut_ptr(), value.as_ptr(), 1, modulus.as_ptr(), &n0, words) + rscrypto_rsa_bn_sqr8x_mont_x86_64_elf( + value.as_mut_ptr(), + value.as_ptr(), + 1, + modulus.as_ptr(), + &raw const n0, + words, + ) }; } @@ -134,7 +146,7 @@ pub(super) fn mont_mul_cios_words_in_place_left( left.as_ptr(), right.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ) }; diff --git a/src/auth/scrypt.rs b/src/auth/scrypt.rs index d60879ac..deb05c2b 100644 --- a/src/auth/scrypt.rs +++ b/src/auth/scrypt.rs @@ -48,12 +48,6 @@ //! [`crate::Pbkdf2Sha256`] (alloc-free) or the `argon2` / `phc-strings` //! features under the same caveats. -#![allow(clippy::indexing_slicing)] -// `unwrap_used` applies to slice→array conversions whose lengths are fixed -// by construction (BLOCK_SIZE / BLOCK_WORDS slicing); every site is bounded -// by compile-time constants and cannot fail at runtime. -#![allow(clippy::unwrap_used)] - use alloc::vec::Vec; use core::fmt; @@ -102,8 +96,8 @@ const MAX_R_TIMES_P: u64 = (1u64 << 30) - 1; /// use rscrypto::{ScryptParams, auth::scrypt::ScryptError}; /// /// assert_eq!( -/// ScryptParams::new(0, 8, 1).unwrap_err(), -/// ScryptError::InvalidLogN +/// ScryptParams::new(0, 8, 1), +/// Err(ScryptError::InvalidLogN) /// ); /// ``` #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -342,9 +336,8 @@ mod x86_sse2 { debug_assert_eq!(blocks.len() % super::BLOCK_SIZE, 0); for chunk in blocks.chunks_exact_mut(super::BLOCK_SIZE) { let mut words = [0u32; super::BLOCK_WORDS]; - for (src, word) in chunk.chunks_exact(4).zip(words.iter_mut()) { - let arr: [u8; 4] = src.try_into().unwrap(); - *word = u32::from_le_bytes(arr); + for (src, word) in chunk.as_chunks::<4>().0.iter().zip(&mut words) { + *word = u32::from_le_bytes(*src); } for (i, dst) in chunk.chunks_exact_mut(4).enumerate() { dst.copy_from_slice(&words[pivot[i]].to_le_bytes()); @@ -438,8 +431,9 @@ mod x86_sse2 { fn integerify_pivot_low64(chunk: &[u8]) -> u64 { debug_assert_eq!(chunk.len() % super::BLOCK_SIZE, 0); let last = &chunk[chunk.len().strict_sub(super::BLOCK_SIZE)..]; - let lo = u32::from_le_bytes(last[0..4].try_into().unwrap()) as u64; - let hi = u32::from_le_bytes(last[52..56].try_into().unwrap()) as u64; + let words = last.as_chunks::<4>().0; + let lo = u64::from(u32::from_le_bytes(words[0])); + let hi = u64::from(u32::from_le_bytes(words[13])); lo | (hi << 32) } @@ -463,9 +457,12 @@ mod x86_sse2 { block_mix(v_chunk, chunk); } - let n_mask = (n as u64).wrapping_sub(1); + let n_mask = u64::try_from(n) + .expect("scrypt allocation length must fit u64") + .strict_sub(1); for _ in 0..n { - let j = (integerify_pivot_low64(chunk) & n_mask) as usize; + let j = + usize::try_from(integerify_pivot_low64(chunk) & n_mask).expect("masked scrypt ROMix index must fit usize"); let v_start = j.strict_mul(len); xor_into(chunk, &v[v_start..v_start.strict_add(len)], scratch); block_mix(scratch, chunk); @@ -495,7 +492,6 @@ impl SalsaBlock { /// then adds the original input back word-wise. Additions are `u32` /// modular-wraparound per the spec. #[inline(always)] -#[allow(clippy::too_many_lines)] fn salsa20_8(block: &mut SalsaBlock) { let input = block.0; let mut y = block.0; @@ -649,10 +645,13 @@ fn ro_mix(chunk: &mut [SalsaBlock], v: &mut [SalsaBlock], scratch: &mut [SalsaBl // ping-pong; each pair XORs V[j] into the current X-buffer before // BlockMix targets the other buffer. X ends in `chunk`, which the // caller re-serialises for the final PBKDF2 leg. - let n_mask = (n as u64).wrapping_sub(1); + let n_mask = u64::try_from(n) + .expect("scrypt allocation length must fit u64") + .strict_sub(1); for _ in 0..pairs { // Even iteration: X lives in `chunk`; write into `scratch`. - let j = (integerify_low64(&chunk[two_r.strict_sub(1)]) & n_mask) as usize; + let j = usize::try_from(integerify_low64(&chunk[two_r.strict_sub(1)]) & n_mask) + .expect("masked scrypt ROMix index must fit usize"); let v_off = j.strict_mul(two_r); for k in 0..two_r { xor_block_into(&mut chunk[k], &v[v_off.strict_add(k)]); @@ -660,7 +659,8 @@ fn ro_mix(chunk: &mut [SalsaBlock], v: &mut [SalsaBlock], scratch: &mut [SalsaBl block_mix_into(chunk, scratch, r); // Odd iteration: X lives in `scratch`; write into `chunk`. - let j = (integerify_low64(&scratch[two_r.strict_sub(1)]) & n_mask) as usize; + let j = usize::try_from(integerify_low64(&scratch[two_r.strict_sub(1)]) & n_mask) + .expect("masked scrypt ROMix index must fit usize"); let v_off = j.strict_mul(two_r); for k in 0..two_r { xor_block_into(&mut scratch[k], &v[v_off.strict_add(k)]); @@ -671,24 +671,10 @@ fn ro_mix(chunk: &mut [SalsaBlock], v: &mut [SalsaBlock], scratch: &mut [SalsaBl // ─── Zeroisation helpers ──────────────────────────────────────────────────── -#[inline] -fn zeroize_u32_slice_no_fence(words: &mut [u32]) { - let mut chunks = words.chunks_exact_mut(16); - for chunk in &mut chunks { - // SAFETY: chunk has exactly 16 initialized u32s and [u32; 16] has the - // same alignment requirement as u32. - unsafe { core::ptr::write_volatile(chunk.as_mut_ptr().cast::<[u32; 16]>(), [0u32; 16]) }; - } - for w in chunks.into_remainder() { - // SAFETY: w is a valid, aligned, dereferenceable pointer to initialized u32. - unsafe { core::ptr::write_volatile(w, 0) }; - } -} - #[inline] fn zeroize_blocks_no_fence(blocks: &mut [SalsaBlock]) { for block in blocks { - zeroize_u32_slice_no_fence(&mut block.0); + ct::zeroize_words_no_fence(&mut block.0); } } @@ -848,10 +834,9 @@ fn scrypt_hash_portable( .map_err(|_| ScryptError::InvalidOutputLen)?; // Decode byte form into little-endian u32 blocks. - for (block, chunk) in state.b_u32.iter_mut().zip(state.b_bytes.chunks_exact(BLOCK_SIZE)) { - for (word, bytes) in block.0.iter_mut().zip(chunk.chunks_exact(4)) { - let arr: [u8; 4] = bytes.try_into().unwrap(); - *word = u32::from_le_bytes(arr); + for (block, chunk) in state.b_u32.iter_mut().zip(state.b_bytes.as_chunks::().0) { + for (word, bytes) in block.0.iter_mut().zip(chunk.as_chunks::<4>().0) { + *word = u32::from_le_bytes(*bytes); } } @@ -865,8 +850,8 @@ fn scrypt_hash_portable( // Re-serialise the mixed B back into the byte buffer for the final // PBKDF2 leg (the spec treats B as a byte string at this point). - for (block, chunk) in state.b_u32.iter().zip(state.b_bytes.chunks_exact_mut(BLOCK_SIZE)) { - for (word, bytes) in block.0.iter().zip(chunk.chunks_exact_mut(4)) { + for (block, chunk) in state.b_u32.iter().zip(state.b_bytes.as_chunks_mut::().0) { + for (word, bytes) in block.0.iter().zip(chunk.as_chunks_mut::<4>().0) { bytes.copy_from_slice(&word.to_le_bytes()); } } @@ -934,12 +919,13 @@ fn scrypt_hash(params: &ScryptParams, password: &[u8], salt: &[u8], out: &mut [u /// ```rust /// use rscrypto::{Scrypt, ScryptParams}; /// -/// // Small CI-friendly params — production deployments should use -/// let params = ScryptParams::new(10, 8, 1).unwrap(); +/// // Small CI-friendly parameters. +/// let params = ScryptParams::new(10, 8, 1)?; /// /// let mut hash = [0u8; 32]; -/// Scrypt::derive(¶ms, b"password", b"random-salt-1234", &mut hash).unwrap(); +/// Scrypt::derive(¶ms, b"password", b"random-salt-1234", &mut hash)?; /// assert!(Scrypt::verify(¶ms, b"password", b"random-salt-1234", &hash).is_ok()); +/// # Ok::<(), rscrypto::ScryptError>(()) /// ``` #[derive(Debug, Clone, Copy, Default)] pub struct Scrypt; @@ -1134,7 +1120,7 @@ mod password_phc { } impl ApprovedPhc { - pub fn salt(&self) -> &[u8] { + pub(super) fn salt(&self) -> &[u8] { &self.salt[..self.salt_len as usize] } } @@ -1175,7 +1161,8 @@ mod password_phc { if log_n > u8::MAX as u32 { return Err(PhcError::ParamOutOfRange); } - let params = ScryptParams::new(log_n as u8, r, p).map_err(|_| PhcError::ParamOutOfRange)?; + let log_n = u8::try_from(log_n).map_err(|_| PhcError::ParamOutOfRange)?; + let params = ScryptParams::new(log_n, r, p).map_err(|_| PhcError::ParamOutOfRange)?; scrypt_shape(¶ms).map_err(|_| PhcError::ParamOutOfRange)?; if !limits.allows(params) { return Err(PhcError::ParamOutOfRange); @@ -1198,7 +1185,7 @@ mod password_phc { Ok(ApprovedPhc { params, salt, - salt_len: decoded_salt_len as u8, + salt_len: u8::try_from(decoded_salt_len).map_err(|_| PhcError::InvalidLength)?, expected, }) } @@ -1235,20 +1222,21 @@ mod tests { ]; fn small_params() -> ScryptParams { - ScryptParams::new(4, 1, 1).unwrap() + ScryptParams::new(4, 1, 1).expect("small scrypt test profile must be valid") } #[test] fn rfc7914_vector_1_empty_inputs() { let mut output = [0u8; 64]; - Scrypt::derive(&small_params(), b"", b"", &mut output).unwrap(); + Scrypt::derive(&small_params(), b"", b"", &mut output) + .expect("RFC 7914 empty-input vector derivation must succeed"); assert_eq!(output, RFC_V1_EXPECTED); } fn oracle_scrypt(password: &[u8], salt: &[u8], log_n: u8, r: u32, p: u32, output_len: usize) -> alloc::vec::Vec { - let params = scrypt::Params::new(log_n, r, p).unwrap(); + let params = scrypt::Params::new(log_n, r, p).expect("oracle scrypt profile must be valid"); let mut output = vec![0u8; output_len]; - scrypt::scrypt(password, salt, ¶ms, &mut output).unwrap(); + scrypt::scrypt(password, salt, ¶ms, &mut output).expect("oracle scrypt derivation must succeed"); output } @@ -1256,9 +1244,10 @@ mod tests { fn matches_the_oracle_across_shapes_and_output_lengths() { let cases: &[(u8, u32, u32, usize)] = &[(4, 1, 1, 16), (5, 2, 1, 32), (6, 2, 2, 64)]; for &(log_n, r, p, output_len) in cases { - let params = ScryptParams::new(log_n, r, p).unwrap(); + let params = ScryptParams::new(log_n, r, p).expect("differential scrypt profile must be valid"); let mut actual = vec![0u8; output_len]; - Scrypt::derive(¶ms, b"password", b"salty-salty-salt", &mut actual).unwrap(); + Scrypt::derive(¶ms, b"password", b"salty-salty-salt", &mut actual) + .expect("rscrypto scrypt derivation must succeed"); assert_eq!( actual, oracle_scrypt(b"password", b"salty-salty-salt", log_n, r, p, output_len), @@ -1272,11 +1261,13 @@ mod tests { fn sse2_backend_matches_portable() { let cases: &[(u8, u32, u32, usize)] = &[(4, 1, 1, 32), (5, 2, 1, 48), (6, 2, 2, 32), (7, 8, 1, 64)]; for &(log_n, r, p, output_len) in cases { - let params = ScryptParams::new(log_n, r, p).unwrap(); + let params = ScryptParams::new(log_n, r, p).expect("backend differential scrypt profile must be valid"); let mut portable = vec![0u8; output_len]; let mut sse2 = vec![0u8; output_len]; - scrypt_hash_portable(¶ms, b"password", b"salty-salty-salt", &mut portable).unwrap(); - scrypt_hash_x86_sse2(¶ms, b"password", b"salty-salty-salt", &mut sse2).unwrap(); + scrypt_hash_portable(¶ms, b"password", b"salty-salty-salt", &mut portable) + .expect("portable scrypt derivation must succeed"); + scrypt_hash_x86_sse2(¶ms, b"password", b"salty-salty-salt", &mut sse2) + .expect("SSE2 scrypt derivation must succeed"); assert_eq!(sse2, portable); } } @@ -1285,16 +1276,29 @@ mod tests { fn raw_verify_accepts_only_the_exact_inputs() { let params = small_params(); let mut expected = [0u8; 32]; - Scrypt::derive(¶ms, b"password", b"random-salt-1234", &mut expected).unwrap(); + Scrypt::derive(¶ms, b"password", b"random-salt-1234", &mut expected) + .expect("scrypt verification fixture derivation must succeed"); - assert!(Scrypt::verify(¶ms, b"password", b"random-salt-1234", &expected).is_ok()); - assert!(Scrypt::verify(¶ms, b"wrong", b"random-salt-1234", &expected).is_err()); - assert!(Scrypt::verify(¶ms, b"password", b"other-salt-00000", &expected).is_err()); + assert_eq!( + Scrypt::verify(¶ms, b"password", b"random-salt-1234", &expected), + Ok(()) + ); + assert_eq!( + Scrypt::verify(¶ms, b"wrong", b"random-salt-1234", &expected), + Err(VerificationError::new()) + ); + assert_eq!( + Scrypt::verify(¶ms, b"password", b"other-salt-00000", &expected), + Err(VerificationError::new()) + ); for position in 0..expected.len() { let mut tampered = expected; tampered[position] ^= 1; - assert!(Scrypt::verify(¶ms, b"password", b"random-salt-1234", &tampered).is_err()); + assert_eq!( + Scrypt::verify(¶ms, b"password", b"random-salt-1234", &tampered), + Err(VerificationError::new()) + ); } } @@ -1305,7 +1309,7 @@ mod tests { assert_eq!(ScryptParams::new(4, 0, 1), Err(ScryptError::InvalidR)); assert_eq!(ScryptParams::new(4, 1, 0), Err(ScryptError::InvalidP)); assert_eq!(ScryptParams::new(4, 1 << 15, 1 << 15), Err(ScryptError::InvalidP)); - assert!(ScryptParams::new(4, 1, 1).is_ok()); + assert_eq!(ScryptParams::new(4, 1, 1), Ok(ScryptParams { log_n: 4, r: 1, p: 1 })); } #[test] @@ -1320,7 +1324,7 @@ mod tests { #[cfg(target_pointer_width = "64")] #[test] fn derive_rejects_impossible_memory_shape_before_allocation() { - let params = ScryptParams::new(63, 1 << 20, 1).unwrap(); + let params = ScryptParams::new(63, 1 << 20, 1).expect("overflow regression profile is structurally valid"); let mut output = [0u8; 32]; assert_eq!( Scrypt::derive(¶ms, b"password", b"salt", &mut output), @@ -1381,14 +1385,14 @@ mod tests { fn encode(params: ScryptParams, password: &[u8], salt: &[u8]) -> alloc::string::String { let mut verifier = [0u8; PASSWORD_OUTPUT_LEN]; - Scrypt::derive(¶ms, password, salt, &mut verifier).unwrap(); + Scrypt::derive(¶ms, password, salt, &mut verifier).expect("scrypt PHC fixture derivation must succeed"); password_phc::encode(params, salt, &verifier) } #[test] fn canonical_password_record_round_trips() { let params = small_params(); - let password = ScryptPassword::new(params).unwrap(); + let password = ScryptPassword::new(params).expect("small scrypt password profile must be valid"); let encoded = encode(params, b"password", &[0xaa; 16]); assert!(encoded.starts_with("$scrypt$ln=4,r=1,p=1$")); @@ -1396,13 +1400,16 @@ mod tests { password.verify_password(b"password", &encoded), Ok(PasswordStatus::Current) ); - assert!(password.verify_password(b"wrong", &encoded).is_err()); + assert_eq!( + password.verify_password(b"wrong", &encoded), + Err(VerificationError::new()) + ); } #[test] fn accepted_older_profile_requests_rehash() { - let generation = ScryptParams::new(5, 1, 1).unwrap(); - let password = ScryptPassword::new(generation).unwrap(); + let generation = ScryptParams::new(5, 1, 1).expect("rehash target profile must be valid"); + let password = ScryptPassword::new(generation).expect("rehash target resource shape must be valid"); let encoded = encode(small_params(), b"password", &[0xbb; 16]); assert_eq!( @@ -1414,7 +1421,7 @@ mod tests { #[test] fn accepted_noncurrent_salt_length_requests_rehash() { let params = small_params(); - let password = ScryptPassword::new(params).unwrap(); + let password = ScryptPassword::new(params).expect("small scrypt password profile must be valid"); let encoded = encode(params, b"password", &[0xbb; 8]); assert_eq!( @@ -1443,8 +1450,8 @@ mod tests { fn limits_bound_both_memory_and_work() { let limits = ScryptVerificationLimits::for_profile(small_params()); assert!(limits.allows(small_params())); - assert!(!limits.allows(ScryptParams::new(5, 1, 1).unwrap())); - assert!(!limits.allows(ScryptParams::new(4, 1, 2).unwrap())); + assert!(!limits.allows(ScryptParams::new(5, 1, 1).expect("memory-limit profile must be valid"))); + assert!(!limits.allows(ScryptParams::new(4, 1, 2).expect("work-limit profile must be valid"))); } #[test] @@ -1453,24 +1460,41 @@ mod tests { let salt = "AAAAAAAAAAAAAAAAAAAAAA"; let hash = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; let cases = [ - format!("$argon2id$v=19$m=32,t=2,p=1$${salt}$${hash}"), - format!("$scrypt$v=1$ln=4,r=1,p=1$${salt}$${hash}"), - format!("$scrypt$r=1,ln=4,p=1$${salt}$${hash}"), - format!("$scrypt$ln=4,ln=4,p=1$${salt}$${hash}"), - format!("$scrypt$ln=4,r=1,x=1$${salt}$${hash}"), - format!("$scrypt$ln=04,r=1,p=1$${salt}$${hash}"), + ( + format!("$argon2id$v=19$m=32,t=2,p=1${salt}${hash}"), + PhcError::AlgorithmMismatch, + ), + ( + format!("$scrypt$v=1$ln=4,r=1,p=1${salt}${hash}"), + PhcError::UnsupportedVersion, + ), + (format!("$scrypt$r=1,ln=4,p=1${salt}${hash}"), PhcError::DuplicateParam), + (format!("$scrypt$ln=4,ln=4,p=1${salt}${hash}"), PhcError::DuplicateParam), + (format!("$scrypt$ln=4,r=1,x=1${salt}${hash}"), PhcError::UnknownParam), + ( + format!("$scrypt$ln=04,r=1,p=1${salt}${hash}"), + PhcError::MalformedParams, + ), ]; - for encoded in cases { - assert!(password_phc::approve(&encoded, limits).is_err(), "{encoded}"); + for (encoded, expected) in cases { + assert_eq!( + password_phc::approve(&encoded, limits).err(), + Some(expected), + "{encoded}" + ); } } #[cfg(feature = "getrandom")] #[test] fn generated_records_use_fresh_salts() { - let password = ScryptPassword::new(small_params()).unwrap(); - let first = password.hash_password(b"password").unwrap(); - let second = password.hash_password(b"password").unwrap(); + let password = ScryptPassword::new(small_params()).expect("small scrypt password profile must be valid"); + let first = password + .hash_password(b"password") + .expect("first scrypt password record must be generated"); + let second = password + .hash_password(b"password") + .expect("second scrypt password record must be generated"); assert_ne!(first, second); assert_eq!( diff --git a/src/auth/x25519.rs b/src/auth/x25519.rs index e0b9c4a9..1f8a35ff 100644 --- a/src/auth/x25519.rs +++ b/src/auth/x25519.rs @@ -281,7 +281,6 @@ impl X25519SecretKey { X25519SharedSecret::diffie_hellman(self, public) } - #[allow(dead_code)] #[cfg(any( not(any( all( @@ -352,7 +351,6 @@ impl Drop for X25519SecretKey { #[derive(Clone, Copy)] pub struct X25519PublicKey { bytes: [u8; Self::LENGTH], - #[allow(dead_code)] #[cfg(any( not(any( all( @@ -362,8 +360,7 @@ pub struct X25519PublicKey { ), all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) )), - miri, - test + miri ))] u: FieldElement, } @@ -389,8 +386,7 @@ impl X25519PublicKey { ), all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) )), - miri, - test + miri ))] u: decode_u_coordinate(&bytes), } @@ -405,7 +401,6 @@ impl X25519PublicKey { #[inline] #[must_use] - #[allow(dead_code)] #[cfg(any( not(any( all( @@ -430,8 +425,7 @@ impl X25519PublicKey { ), all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) )), - miri, - test + miri ))] u, } @@ -596,7 +590,6 @@ impl Drop for X25519SharedSecret { } } -#[allow(clippy::indexing_slicing)] #[must_use] #[cfg(any( not(any( @@ -715,7 +708,7 @@ fn decode_u_coordinate(bytes: &[u8; POINT_LENGTH]) -> FieldElement { acc_bits = acc_bits.wrapping_add(8); } - *limb = (acc & u128::from(MASK51)) as u64; + *limb = u64::try_from(acc & u128::from(MASK51)).expect("masked field limb fits u64"); acc >>= RADIX_BITS; acc_bits = acc_bits.wrapping_sub(RADIX_BITS); } @@ -760,18 +753,18 @@ mod tests { fn scalar(seed: u8) -> [u8; POINT_LENGTH] { let mut out = [0u8; POINT_LENGTH]; - for (index, byte) in out.iter_mut().enumerate() { - *byte = seed.wrapping_mul(37).wrapping_add((index as u8).wrapping_mul(19)); + for (index, byte) in (0u8..).zip(&mut out) { + *byte = seed.wrapping_mul(37).wrapping_add(index.wrapping_mul(19)); } out } fn peer(seed: u8) -> [u8; POINT_LENGTH] { let mut out = [0u8; POINT_LENGTH]; - for (index, byte) in out.iter_mut().enumerate() { + for (index, byte) in (0u8..).zip(&mut out) { *byte = seed .wrapping_mul(53) - .wrapping_add((index as u8).wrapping_mul(11)) + .wrapping_add(index.wrapping_mul(11)) .wrapping_add(7); } out[POINT_LENGTH - 1] |= 0x80; diff --git a/src/auth/x25519/aarch64_asm.rs b/src/auth/x25519/aarch64_asm.rs index 3d6ed5d6..fbef2588 100644 --- a/src/auth/x25519/aarch64_asm.rs +++ b/src/auth/x25519/aarch64_asm.rs @@ -4,8 +4,6 @@ //! byte backends. This module owns the ABI boundary and keeps the public //! rscrypto API semantics in `x25519.rs`. -#![allow(unsafe_code)] - use core::arch::global_asm; use super::POINT_LENGTH; diff --git a/src/auth/x25519/x86_64_asm.rs b/src/auth/x25519/x86_64_asm.rs index 0df729e3..709a13ec 100644 --- a/src/auth/x25519/x86_64_asm.rs +++ b/src/auth/x25519/x86_64_asm.rs @@ -4,8 +4,6 @@ //! byte backends. This module owns the ABI boundary and keeps the public //! rscrypto API semantics in `x25519.rs`. -#![allow(unsafe_code)] - use core::arch::global_asm; use super::POINT_LENGTH; diff --git a/src/backend/ascon.rs b/src/backend/ascon.rs index cb1b3b26..9a7327e8 100644 --- a/src/backend/ascon.rs +++ b/src/backend/ascon.rs @@ -67,8 +67,7 @@ pub(crate) fn permute_12_portable(s: &mut [u64; 5]) { /// /// Fully unrolled for the same register-allocation benefits as /// [`permute_12_portable`]. -#[cfg(any(feature = "ascon-aead", test))] -#[cfg_attr(test, allow(dead_code))] +#[cfg(feature = "ascon-aead")] #[inline(always)] pub(crate) fn permute_8_portable(s: &mut [u64; 5]) { let mut x0 = s[0]; diff --git a/src/backend/cache.rs b/src/backend/cache.rs index 59027bcd..6034b496 100644 --- a/src/backend/cache.rs +++ b/src/backend/cache.rs @@ -106,7 +106,7 @@ impl AtomicOnceCache { core::hint::spin_loop(); } } - _ => unreachable!("atomic cache state is private and has three values"), + _ => core::hint::spin_loop(), } } } @@ -116,7 +116,7 @@ impl AtomicOnceCache { /// /// Building block for dispatcher caching with proper synchronization. /// See module documentation for platform-specific behavior. -pub struct OnceCache { +pub(crate) struct OnceCache { #[cfg(feature = "std")] inner: std::sync::OnceLock, @@ -130,7 +130,7 @@ pub struct OnceCache { impl OnceCache { /// Create a new empty cache. #[must_use] - pub const fn new() -> Self { + pub(crate) const fn new() -> Self { Self { #[cfg(feature = "std")] inner: std::sync::OnceLock::new(), @@ -151,7 +151,7 @@ impl OnceCache { /// /// Returns the cached value by copy (since T is Copy). #[inline] - pub fn get_or_init(&self, f: impl FnOnce() -> T) -> T { + pub(crate) fn get_or_init(&self, f: impl FnOnce() -> T) -> T { #[cfg(feature = "std")] { *self.inner.get_or_init(f) @@ -213,13 +213,13 @@ mod tests { } #[cfg(feature = "std")] - #[allow(clippy::std_instead_of_core, clippy::std_instead_of_alloc)] mod threading_tests { - use std::{ + use alloc::{boxed::Box, vec::Vec}; + use core::{ + panic::AssertUnwindSafe, sync::atomic::{AtomicUsize, Ordering}, - thread, - vec::Vec, }; + use std::thread; use super::*; @@ -243,7 +243,7 @@ mod tests { .collect(); for handle in handles { - handle.join().unwrap(); + handle.join().expect("cache worker thread must not panic"); } assert_eq!(CALL_COUNT.load(Ordering::SeqCst), 1); @@ -253,20 +253,19 @@ mod tests { fn test_atomic_once_cache_recovers_after_initializer_panic() { let cache = AtomicOnceCache::::new(); - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - cache.get_or_init(|| panic!("controlled initializer panic")); + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + cache.get_or_init(|| std::panic::resume_unwind(Box::new("controlled initializer panic"))); })); - assert!(result.is_err()); + result.expect_err("panicking initializer must unwind"); assert_eq!(cache.get_or_init(|| 23), 23); } #[test] fn test_atomic_once_cache_waiter_retries_after_initializer_panic() { - use std::{ - sync::{Arc, Barrier, mpsc}, - time::Duration, - }; + use alloc::sync::Arc; + use core::time::Duration; + use std::sync::{Barrier, mpsc}; let cache = Arc::new(AtomicOnceCache::::new()); let initializer_entered = Arc::new(Barrier::new(2)); @@ -276,11 +275,11 @@ mod tests { let panicking_entered = Arc::clone(&initializer_entered); let panicking_release = Arc::clone(&release_initializer); let panicking = thread::spawn(move || { - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + std::panic::catch_unwind(AssertUnwindSafe(|| { panicking_cache.get_or_init(|| { panicking_entered.wait(); panicking_release.wait(); - panic!("controlled initializer panic"); + std::panic::resume_unwind(Box::new("controlled initializer panic")) }); })) }); @@ -289,13 +288,23 @@ mod tests { let waiting_cache = Arc::clone(&cache); let (sender, receiver) = mpsc::channel(); let waiting = thread::spawn(move || { - sender.send(waiting_cache.get_or_init(|| 29)).unwrap(); + sender + .send(waiting_cache.get_or_init(|| 29)) + .expect("cache waiter result receiver must remain connected"); }); release_initializer.wait(); - assert!(panicking.join().unwrap().is_err()); - assert_eq!(receiver.recv_timeout(Duration::from_secs(2)).unwrap(), 29); - waiting.join().unwrap(); + panicking + .join() + .expect("panicking cache thread must unwind only inside catch_unwind") + .expect_err("cache initializer must panic"); + assert_eq!( + receiver + .recv_timeout(Duration::from_secs(2)) + .expect("cache waiter must publish its result before the timeout"), + 29 + ); + waiting.join().expect("cache waiter thread must not panic"); } #[test] @@ -318,7 +327,7 @@ mod tests { .collect(); for handle in handles { - handle.join().unwrap(); + handle.join().expect("cache worker thread must not panic"); } // Selector called exactly once diff --git a/src/backend/curve25519.rs b/src/backend/curve25519.rs index bf15b085..c30a6e38 100644 --- a/src/backend/curve25519.rs +++ b/src/backend/curve25519.rs @@ -32,8 +32,7 @@ const MODULUS_N: u64 = MASK51; /// chains. const SUB_BIAS_0: u64 = ((1u64 << RADIX_BITS) - 19).wrapping_mul(16); const SUB_BIAS_N: u64 = MASK51.wrapping_mul(16); -#[allow(dead_code)] -#[cfg(any(feature = "ed25519", feature = "x25519"))] +#[cfg(feature = "ed25519")] const SQRT_M1: FieldElement = FieldElement::from_limbs([ 1_718_705_420_411_056, 234_908_883_556_509, @@ -63,7 +62,6 @@ impl FieldElement { /// Construct a small field element without reduction. #[inline] #[must_use] - #[allow(dead_code)] pub(crate) const fn from_small(value: u64) -> Self { Self([value, 0, 0, 0, 0]) } @@ -71,7 +69,6 @@ impl FieldElement { /// Borrow the raw radix-51 limbs. #[inline] #[must_use] - #[allow(dead_code)] #[cfg(any(feature = "ed25519", feature = "x25519"))] pub(crate) const fn limbs(&self) -> &[u64; FIELD_LIMBS] { &self.0 @@ -180,7 +177,6 @@ impl FieldElement { /// Negate the field element modulo `2^255 - 19`. #[inline] #[must_use] - #[allow(dead_code)] #[cfg(any(feature = "ed25519", feature = "x25519"))] pub(crate) fn neg(&self) -> Self { Self::ZERO.sub(self) @@ -217,8 +213,7 @@ impl FieldElement { /// Decode a canonical 32-byte field element. #[must_use] - #[allow(dead_code)] - #[cfg(any(feature = "ed25519", feature = "x25519"))] + #[cfg(feature = "ed25519")] pub(crate) fn from_bytes(bytes: &[u8; 32]) -> Option { let mut acc = 0u128; let mut acc_bits = 0u32; @@ -235,7 +230,7 @@ impl FieldElement { } } - *limb = (acc & u128::from(MASK51)) as u64; + *limb = u64::try_from(acc & u128::from(MASK51)).expect("masked field limb fits u64"); acc >>= RADIX_BITS; acc_bits = acc_bits.wrapping_sub(RADIX_BITS); } @@ -267,7 +262,7 @@ impl FieldElement { while acc_bits >= 8 { if let Some(byte) = out_iter.next() { - *byte = acc as u8; + *byte = u8::try_from(acc & 0xff).expect("masked field byte fits u8"); } acc >>= 8; acc_bits = acc_bits.wrapping_sub(8); @@ -275,7 +270,7 @@ impl FieldElement { } if let Some(byte) = out_iter.next() { - *byte = acc as u8; + *byte = u8::try_from(acc & 0xff).expect("masked field byte fits u8"); } out @@ -283,7 +278,6 @@ impl FieldElement { /// Return `true` when the canonical field element is zero. #[must_use] - #[allow(dead_code)] #[cfg(any(feature = "ed25519", feature = "x25519"))] pub(crate) fn is_zero(&self) -> bool { self.normalize().0.iter().all(|&limb| limb == 0) @@ -291,8 +285,22 @@ impl FieldElement { /// Branchless conditional swap at the Rust source level. #[inline] - #[cfg(feature = "x25519")] - #[allow(dead_code)] + #[cfg(all( + feature = "x25519", + any( + feature = "diag", + test, + miri, + not(any( + all( + target_arch = "aarch64", + any(target_os = "macos", target_os = "linux"), + not(feature = "portable-only") + ), + all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) + )) + ) + ))] pub(crate) fn conditional_swap(lhs: &mut Self, rhs: &mut Self, swap: u8) { let mask = 0u64.wrapping_sub(u64::from(swap & 1)); for (lhs_limb, rhs_limb) in lhs.0.iter_mut().zip(rhs.0.iter_mut()) { @@ -304,16 +312,14 @@ impl FieldElement { /// Return the low-bit sign of the canonical encoding. #[must_use] - #[allow(dead_code)] - #[cfg(any(feature = "ed25519", feature = "x25519"))] + #[cfg(feature = "ed25519")] pub(crate) fn is_negative(&self) -> bool { ((*self).to_bytes()[0] & 1) == 1 } /// Square root in `GF(2^255 - 19)` when one exists. #[must_use] - #[allow(dead_code)] - #[cfg(any(feature = "ed25519", feature = "x25519"))] + #[cfg(feature = "ed25519")] pub(crate) fn sqrt(&self) -> Option { let normalized = self.normalize(); self @@ -360,16 +366,14 @@ impl FieldElement { } #[must_use] - #[allow(dead_code)] - #[cfg(any(feature = "ed25519", feature = "x25519"))] + #[cfg(feature = "ed25519")] fn pow_p58(&self) -> Self { let (t19, _) = self.pow22501(); self.mul(&t19.pow2k(2)) } #[must_use] - #[allow(dead_code)] - #[cfg(any(feature = "ed25519", feature = "x25519"))] + #[cfg(feature = "ed25519")] pub(crate) fn sqrt_ratio_i(&self, denominator: &Self) -> Option { let numerator = self.normalize(); let denominator = denominator.normalize(); @@ -454,7 +458,7 @@ fn reduce_wide(wide: [u128; FIELD_LIMBS]) -> [u64; FIELD_LIMBS] { h0 = h0.wrapping_add((h4 >> RADIX_BITS).wrapping_mul(19)); h4 &= mask; - [h0 as u64, h1 as u64, h2 as u64, h3 as u64, h4 as u64] + [h0, h1, h2, h3, h4].map(|limb| u64::try_from(limb).expect("reduced field limb fits u64")) } /// Single-round carry propagation on u64 limbs (sub output, normalize). @@ -518,7 +522,7 @@ mod tests { fn from_u128(mut value: u128) -> FieldElement { let mut limbs = [0u64; 5]; for limb in &mut limbs { - *limb = (value & u128::from(MASK51)) as u64; + *limb = u64::try_from(value & u128::from(MASK51)).expect("masked test limb fits u64"); value >>= 51; } FieldElement::from_limbs(limbs) diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 4f03f7db..d64af482 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -5,7 +5,7 @@ //! so algorithms can reuse implementation building blocks without creating //! public-surface or feature-graph coupling. #[cfg(any(feature = "ascon-hash", feature = "ascon-aead"))] -pub mod ascon; +pub(crate) mod ascon; #[cfg(any( feature = "crc16", feature = "crc24", @@ -19,13 +19,30 @@ pub mod ascon; not(all(target_arch = "aarch64", target_os = "macos")) ), feature = "blake3", - feature = "ascon-hash", + all(feature = "ascon-hash", feature = "diag"), feature = "xxh3", feature = "aes-gcm", feature = "aes-gcm-siv", feature = "chacha20poly1305", feature = "xchacha20poly1305" ))] -pub mod cache; -#[cfg(any(feature = "ed25519", feature = "x25519"))] -pub mod curve25519; +pub(crate) mod cache; +#[cfg(any( + feature = "ed25519", + all( + feature = "x25519", + any( + test, + miri, + not(any( + all( + target_arch = "aarch64", + any(target_os = "macos", target_os = "linux"), + not(feature = "portable-only") + ), + all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) + )) + ) + ) +))] +pub(crate) mod curve25519; diff --git a/src/checksum/common/clmul.rs b/src/checksum/common/clmul.rs index 9c6cea00..96e582f2 100644 --- a/src/checksum/common/clmul.rs +++ b/src/checksum/common/clmul.rs @@ -42,7 +42,7 @@ const fn clmul64(a: u64, b: u64) -> (u64, u64) { lo ^= b; } else { lo ^= b << i; - hi ^= b >> (64 - i); + hi ^= b >> 64u32.strict_sub(i); } } i = i.strict_add(1); @@ -79,7 +79,7 @@ pub(crate) const fn reduce128(hi: u64, lo: u64, poly: u64) -> u64 { } else { // XOR poly shifted by i result_lo ^= poly << i; - result_hi ^= (poly >> (64 - i)) | (1 << i); + result_hi ^= (poly >> 64i32.strict_sub(i)) | (1 << i); } } i = i.strict_sub(1); @@ -176,14 +176,14 @@ pub(crate) const fn fold16_coeff_for_bytes(reflected_poly: u64, shift_bytes: u32 } let normal = normal_poly(reflected_poly); - let d = shift_bytes * 8; + let d = shift_bytes.strict_mul(8); // `d >= 8`, so `d - 1` is valid. - (fold_k(normal, d - 1), fold_k(normal, d + 63)) + (fold_k(normal, d.strict_sub(1)), fold_k(normal, d.strict_add(63))) } impl Crc64ClmulConstants { #[must_use] - pub const fn new(reflected_poly: u64) -> Self { + const fn new(reflected_poly: u64) -> Self { let poly = reciprocal_poly(reflected_poly); let normal = normal_poly(reflected_poly); let mu = compute_tikv_mu(poly); @@ -227,7 +227,7 @@ const fn compute_tikv_mu(poly: u64) -> u64 { let mut i: u32 = 1; while i <= k { let p_i = (poly >> i) & 1; - let q_j = (inv >> (k - i)) & 1; + let q_j = (inv >> k.strict_sub(i)) & 1; s ^= p_i & q_j; i = i.strict_add(1); } @@ -262,23 +262,29 @@ pub(crate) const CRC64_NVME_CLMUL: Crc64ClmulConstants = Crc64ClmulConstants::ne /// - `combine_8way`: merge coefficients for 8-way (x86_64) #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] #[derive(Clone, Copy, Debug)] -#[allow(dead_code)] // Field subsets vary by architecture (x86_64 vs aarch64 stream widths). pub(crate) struct Crc64StreamConstants { /// 2-way fold coefficient (256B = 2×128B). pub fold_256b: (u64, u64), /// 3-way fold coefficient (384B = 3×128B). + #[cfg(target_arch = "aarch64")] pub fold_384b: (u64, u64), /// 4-way fold coefficient (512B = 4×128B). + #[cfg(target_arch = "x86_64")] pub fold_512b: (u64, u64), /// 7-way fold coefficient (896B = 7×128B). + #[cfg(target_arch = "x86_64")] pub fold_896b: (u64, u64), /// 8-way fold coefficient (1024B = 8×128B). + #[cfg(target_arch = "x86_64")] pub fold_1024b: (u64, u64), /// 4-way combine coefficients: shifts by 384B, 256B, 128B. + #[cfg(target_arch = "x86_64")] pub combine_4way: [(u64, u64); 3], /// 7-way combine coefficients: shifts by 768B, 640B, 512B, 384B, 256B, 128B. + #[cfg(target_arch = "x86_64")] pub combine_7way: [(u64, u64); 6], /// 8-way combine coefficients: shifts by 896B, 768B, 640B, 512B, 384B, 256B, 128B. + #[cfg(target_arch = "x86_64")] pub combine_8way: [(u64, u64); 7], } @@ -286,18 +292,24 @@ pub(crate) struct Crc64StreamConstants { impl Crc64StreamConstants { /// Compute all multi-stream folding constants for a given polynomial. #[must_use] - pub const fn new(reflected_poly: u64) -> Self { + const fn new(reflected_poly: u64) -> Self { Self { fold_256b: fold16_coeff_for_bytes(reflected_poly, 256), + #[cfg(target_arch = "aarch64")] fold_384b: fold16_coeff_for_bytes(reflected_poly, 384), + #[cfg(target_arch = "x86_64")] fold_512b: fold16_coeff_for_bytes(reflected_poly, 512), + #[cfg(target_arch = "x86_64")] fold_896b: fold16_coeff_for_bytes(reflected_poly, 896), + #[cfg(target_arch = "x86_64")] fold_1024b: fold16_coeff_for_bytes(reflected_poly, 1024), + #[cfg(target_arch = "x86_64")] combine_4way: [ fold16_coeff_for_bytes(reflected_poly, 384), fold16_coeff_for_bytes(reflected_poly, 256), fold16_coeff_for_bytes(reflected_poly, 128), ], + #[cfg(target_arch = "x86_64")] combine_7way: [ fold16_coeff_for_bytes(reflected_poly, 768), fold16_coeff_for_bytes(reflected_poly, 640), @@ -306,6 +318,7 @@ impl Crc64StreamConstants { fold16_coeff_for_bytes(reflected_poly, 256), fold16_coeff_for_bytes(reflected_poly, 128), ], + #[cfg(target_arch = "x86_64")] combine_8way: [ fold16_coeff_for_bytes(reflected_poly, 896), fold16_coeff_for_bytes(reflected_poly, 768), diff --git a/src/checksum/common/combine.rs b/src/checksum/common/combine.rs index d15ee498..269102c3 100644 --- a/src/checksum/common/combine.rs +++ b/src/checksum/common/combine.rs @@ -17,7 +17,6 @@ // SAFETY: All array indexing in this module uses bounded loop indices (0..N where N is the // array size). Clippy cannot prove this in const fn contexts, but the bounds are statically // guaranteed by the loop conditions. -#![allow(clippy::indexing_slicing)] // Unified GF(2) matrix macro @@ -55,12 +54,12 @@ macro_rules! define_gf2_combine { ) => { #[doc = $doc_matrix] #[derive(Clone, Copy)] - pub struct $Name([$T; $DIM]); + pub(in crate::checksum) struct $Name([$T; $DIM]); impl $Name { /// Create the identity matrix. #[must_use] - pub const fn identity() -> Self { + pub(in crate::checksum) const fn identity() -> Self { let mut m = [0 as $T; $DIM]; let mut i: u32 = 0; while i < $DIM as u32 { @@ -73,7 +72,7 @@ macro_rules! define_gf2_combine { /// Multiply matrix by a vector. #[inline] #[must_use] - pub const fn mul_vec(self, vec: $T) -> $T { + pub(in crate::checksum) const fn mul_vec(self, vec: $T) -> $T { let mut result = 0 as $T; let mut i: u32 = 0; while i < $DIM as u32 { @@ -87,7 +86,7 @@ macro_rules! define_gf2_combine { /// Multiply two matrices. #[must_use] - pub const fn mul_mat(self, other: Self) -> Self { + pub(in crate::checksum) const fn mul_mat(self, other: Self) -> Self { let mut result = [0 as $T; $DIM]; let mut i: u32 = 0; while i < $DIM as u32 { @@ -100,14 +99,14 @@ macro_rules! define_gf2_combine { /// Square the matrix. #[inline] #[must_use] - pub const fn square(self) -> Self { + pub(in crate::checksum) const fn square(self) -> Self { self.mul_mat(self) } } #[doc = $doc_shift1] #[must_use] - pub const fn $shift1_fn(poly: $T) -> $Name { + const fn $shift1_fn(poly: $T) -> $Name { let mut m = [0 as $T; $DIM]; m[0] = poly; let mut j: u32 = 1; @@ -120,7 +119,7 @@ macro_rules! define_gf2_combine { #[doc = $doc_shift8] #[must_use] - pub const fn $shift8_fn(poly: $T) -> $Name { + pub(in crate::checksum) const fn $shift8_fn(poly: $T) -> $Name { let shift1 = $shift1_fn(poly); let shift2 = shift1.square(); let shift4 = shift2.square(); @@ -129,7 +128,7 @@ macro_rules! define_gf2_combine { #[doc = $doc_combine] #[must_use] - pub const fn $combine_fn(crc_a: $T, crc_b: $T, len_b: usize, shift8_matrix: $Name) -> $T { + pub(in crate::checksum) const fn $combine_fn(crc_a: $T, crc_b: $T, len_b: usize, shift8_matrix: $Name) -> $T { if len_b == 0 { return crc_a; } @@ -170,12 +169,12 @@ macro_rules! define_gf2_combine { ) => { #[doc = $doc_matrix] #[derive(Clone, Copy)] - pub struct $Name([$T; $DIM]); + pub(in crate::checksum) struct $Name([$T; $DIM]); impl $Name { /// Create the identity matrix. #[must_use] - pub const fn identity() -> Self { + const fn identity() -> Self { let mut m = [0 as $T; $DIM]; let mut i: u32 = 0; while i < $DIM as u32 { @@ -188,7 +187,7 @@ macro_rules! define_gf2_combine { /// Multiply matrix by a vector. #[inline] #[must_use] - pub const fn mul_vec(self, vec: $T) -> $T { + const fn mul_vec(self, vec: $T) -> $T { let mut result = 0 as $T; let mut i: u32 = 0; while i < $DIM as u32 { @@ -202,7 +201,7 @@ macro_rules! define_gf2_combine { /// Multiply two matrices. #[must_use] - pub const fn mul_mat(self, other: Self) -> Self { + const fn mul_mat(self, other: Self) -> Self { let mut result = [0 as $T; $DIM]; let mut i: u32 = 0; while i < $DIM as u32 { @@ -215,14 +214,14 @@ macro_rules! define_gf2_combine { /// Square the matrix. #[inline] #[must_use] - pub const fn square(self) -> Self { + const fn square(self) -> Self { self.mul_mat(self) } } #[doc = $doc_shift1] #[must_use] - pub const fn $shift1_fn(poly: $T) -> $Name { + const fn $shift1_fn(poly: $T) -> $Name { let mut m = [0 as $T; $DIM]; m[0] = poly; let mut j: u32 = 1; @@ -235,7 +234,7 @@ macro_rules! define_gf2_combine { #[doc = $doc_shift8] #[must_use] - pub const fn $shift8_fn(poly: $T) -> $Name { + pub(in crate::checksum) const fn $shift8_fn(poly: $T) -> $Name { let shift1 = $shift1_fn(poly); let shift2 = shift1.square(); let shift4 = shift2.square(); @@ -246,7 +245,13 @@ macro_rules! define_gf2_combine { /// This works for any init/xorout as long as the caller supplies /// `init_xorout = init ^ xorout` for the CRC variant being combined. #[must_use] - pub const fn $combine_fn(crc_a: $T, crc_b: $T, len_b: usize, shift8_matrix: $Name, init_xorout: $T) -> $T { + pub(in crate::checksum) const fn $combine_fn( + crc_a: $T, + crc_b: $T, + len_b: usize, + shift8_matrix: $Name, + init_xorout: $T, + ) -> $T { if len_b == 0 { return crc_a; } @@ -288,15 +293,15 @@ macro_rules! define_gf2_combine { ) => { #[doc = $doc_matrix] #[derive(Clone, Copy)] - pub struct $Name([$T; $DIM]); + pub(in crate::checksum) struct $Name([$T; $DIM]); impl $Name { /// Bit mask that keeps only the low `DIM` bits of the backing type. - pub const MASK: $T = $MASK; + const MASK: $T = $MASK; /// Create the identity matrix. #[must_use] - pub const fn identity() -> Self { + const fn identity() -> Self { let mut m = [0 as $T; $DIM]; let mut i: u32 = 0; while i < $DIM as u32 { @@ -309,7 +314,7 @@ macro_rules! define_gf2_combine { /// Multiply matrix by a vector (low bits only). #[inline] #[must_use] - pub const fn mul_vec(self, vec: $T) -> $T { + const fn mul_vec(self, vec: $T) -> $T { let vec = vec & Self::MASK; let mut result = 0 as $T; let mut i: u32 = 0; @@ -324,7 +329,7 @@ macro_rules! define_gf2_combine { /// Multiply two matrices. #[must_use] - pub const fn mul_mat(self, other: Self) -> Self { + const fn mul_mat(self, other: Self) -> Self { let mut result = [0 as $T; $DIM]; let mut i: u32 = 0; while i < $DIM as u32 { @@ -337,14 +342,14 @@ macro_rules! define_gf2_combine { /// Square the matrix. #[inline] #[must_use] - pub const fn square(self) -> Self { + const fn square(self) -> Self { self.mul_mat(self) } } #[doc = $doc_shift1] #[must_use] - pub const fn $shift1_fn(poly: $T) -> $Name { + const fn $shift1_fn(poly: $T) -> $Name { let poly = poly & $Name::MASK; let mut m = [0 as $T; $DIM]; @@ -361,7 +366,7 @@ macro_rules! define_gf2_combine { #[doc = $doc_shift8] #[must_use] - pub const fn $shift8_fn(poly: $T) -> $Name { + pub(in crate::checksum) const fn $shift8_fn(poly: $T) -> $Name { let shift1 = $shift1_fn(poly); let shift2 = shift1.square(); let shift4 = shift2.square(); @@ -372,7 +377,13 @@ macro_rules! define_gf2_combine { /// This works for any init/xorout as long as the caller supplies /// `init_xorout = init ^ xorout` for the CRC variant being combined. #[must_use] - pub const fn $combine_fn(crc_a: $T, crc_b: $T, len_b: usize, shift8_matrix: $Name, init_xorout: $T) -> $T { + pub(in crate::checksum) const fn $combine_fn( + crc_a: $T, + crc_b: $T, + len_b: usize, + shift8_matrix: $Name, + init_xorout: $T, + ) -> $T { if len_b == 0 { return crc_a & $Name::MASK; } @@ -465,7 +476,7 @@ define_gf2_combine! { #[inline] #[must_use] #[cfg(all(feature = "crc32", target_arch = "aarch64"))] -pub const fn pow_shift8_matrix_32(len_bytes: usize, shift8_matrix: Gf2Matrix32) -> Gf2Matrix32 { +pub(in crate::checksum) const fn pow_shift8_matrix_32(len_bytes: usize, shift8_matrix: Gf2Matrix32) -> Gf2Matrix32 { if len_bytes == 0 { return Gf2Matrix32::identity(); } diff --git a/src/checksum/common/kernels.rs b/src/checksum/common/kernels.rs index 6c89e1fc..a6ee6a50 100644 --- a/src/checksum/common/kernels.rs +++ b/src/checksum/common/kernels.rs @@ -1,12 +1,12 @@ //! Shared CRC kernel names. /// Reference (bitwise) kernel name - canonical implementation for verification. -pub const REFERENCE: &str = "reference/bitwise"; +pub(in crate::checksum) const REFERENCE: &str = "reference/bitwise"; /// Portable fallback kernel name (used by all CRC widths). #[cfg(any(feature = "crc32", feature = "crc64"))] -pub const PORTABLE_SLICE16: &str = "portable/slice16"; +pub(in crate::checksum) const PORTABLE_SLICE16: &str = "portable/slice16"; /// Portable slice-by-8 kernel name. #[cfg(any(feature = "crc16", feature = "crc24"))] -pub const PORTABLE_SLICE8: &str = "portable/slice8"; +pub(in crate::checksum) const PORTABLE_SLICE8: &str = "portable/slice8"; diff --git a/src/checksum/common/mod.rs b/src/checksum/common/mod.rs index 15032ae4..54178a67 100644 --- a/src/checksum/common/mod.rs +++ b/src/checksum/common/mod.rs @@ -17,16 +17,46 @@ target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64", - target_arch = "s390x", - target_arch = "riscv64" + target_arch = "s390x" ) ))] -pub mod clmul; -pub mod combine; -pub mod kernels; -pub mod portable; -pub mod prefetch; -pub mod reference; -pub mod tables; +pub(in crate::checksum) mod clmul; +pub(in crate::checksum) mod combine; +pub(in crate::checksum) mod kernels; +pub(in crate::checksum) mod portable; +#[cfg(any( + all( + target_arch = "x86_64", + any(feature = "crc16", feature = "crc24", feature = "crc32", feature = "crc64") + ), + all( + target_arch = "aarch64", + any(feature = "crc16", feature = "crc24", feature = "crc64") + ), + test +))] +pub(in crate::checksum) mod prefetch; +pub(in crate::checksum) mod reference; +pub(in crate::checksum) mod tables; #[cfg(test)] -pub mod tests; +pub(in crate::checksum) mod tests; + +#[inline] +#[cfg(all( + feature = "crc16", + any(target_arch = "powerpc64", target_arch = "riscv64", target_arch = "s390x") +))] +pub(in crate::checksum) const fn low_u16(value: u32) -> u16 { + let [b0, b1, ..] = value.to_le_bytes(); + u16::from_le_bytes([b0, b1]) +} + +#[inline] +#[cfg(all( + any(feature = "crc16", feature = "crc24", feature = "crc32"), + any(target_arch = "powerpc64", target_arch = "riscv64", target_arch = "s390x") +))] +pub(in crate::checksum) const fn low_u32(value: u64) -> u32 { + let [b0, b1, b2, b3, ..] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) +} diff --git a/src/checksum/common/portable.rs b/src/checksum/common/portable.rs index cafbcbc9..ba7dfc43 100644 --- a/src/checksum/common/portable.rs +++ b/src/checksum/common/portable.rs @@ -20,8 +20,6 @@ // - chunks_exact guarantees chunk sizes // - Table indices use `& 0xFF` (0..255) or explicit byte extraction // Clippy cannot prove this in const fn contexts, but bounds are statically guaranteed. -#![allow(clippy::indexing_slicing)] -#![cfg_attr(all(target_arch = "wasm32", target_feature = "simd128"), allow(unsafe_code))] #[cfg(any(feature = "crc16", feature = "crc32", feature = "crc64"))] macro_rules! tail_step { @@ -241,7 +239,7 @@ pub(crate) fn slice8_24(crc: u32, data: &[u8], tables: &[[u32; 256]; 8]) -> u32 /// * `tables` - 8 lookup tables (256 entries each) #[cfg(all(test, feature = "crc32"))] #[inline] -pub fn slice8_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 8]) -> u32 { +fn slice8_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 8]) -> u32 { let (chunks, remainder) = data.as_chunks::<8>(); for chunk in chunks { @@ -275,9 +273,9 @@ pub fn slice8_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 8]) -> u32 { #[inline] fn slice16_32_scalar(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { let (chunks4, remainder) = data.as_chunks::<4>(); - let mut quads = chunks4.chunks_exact(4); + let (quads, trailing_chunks) = chunks4.as_chunks::<4>(); - for quad in quads.by_ref() { + for quad in quads { let a = u32::from_le_bytes(quad[0]) ^ crc; let b = u32::from_le_bytes(quad[1]); let c = u32::from_le_bytes(quad[2]); @@ -302,7 +300,7 @@ fn slice16_32_scalar(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u3 } // Handle a 4-byte tail (one to three u32 chunks) - for chunk in quads.remainder() { + for chunk in trailing_chunks { let val = u32::from_le_bytes(*chunk) ^ crc; crc = tables[3][(val & 0xFF) as usize] ^ tables[2][((val >> 8) & 0xFF) as usize] @@ -325,7 +323,7 @@ fn slice16_32_scalar(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u3 /// * `tables` - 16 lookup tables (256 entries each) #[cfg(all(feature = "crc32", not(all(target_arch = "wasm32", target_feature = "simd128"))))] #[inline] -pub fn slice16_32(crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { +pub(in crate::checksum) fn slice16_32(crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { slice16_32_scalar(crc, data, tables) } @@ -335,7 +333,7 @@ pub fn slice16_32(crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { /// `v128` loads when `target_feature = "simd128"` is enabled. #[cfg(all(feature = "crc32", target_arch = "wasm32", target_feature = "simd128"))] #[inline] -pub fn slice16_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { +pub(in crate::checksum) fn slice16_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { use core::arch::wasm32::v128; let mut ptr = data.as_ptr(); @@ -343,7 +341,7 @@ pub fn slice16_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { while len >= 16 { // SAFETY: `ptr` is within `data` and `len >= 16`. - let v = unsafe { core::ptr::read_unaligned(ptr as *const v128) }; + let v = unsafe { core::ptr::read_unaligned(ptr.cast::()) }; // Extract lanes using WASM SIMD intrinsics (avoids transmute layout assumptions). let a = core::arch::wasm32::u32x4_extract_lane::<0>(v) ^ crc; let b = core::arch::wasm32::u32x4_extract_lane::<1>(v); @@ -390,7 +388,7 @@ pub fn slice16_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { /// * `tables` - 8 lookup tables (256 entries each) #[cfg(all(feature = "crc64", any(target_arch = "x86_64", target_arch = "aarch64", test)))] #[inline] -pub fn slice8_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 8]) -> u64 { +pub(in crate::checksum) fn slice8_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 8]) -> u64 { let (chunks, remainder) = data.as_chunks::<8>(); for chunk in chunks { @@ -423,9 +421,9 @@ pub fn slice8_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 8]) -> u64 { #[inline] fn slice16_64_scalar(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { let (chunks8, remainder) = data.as_chunks::<8>(); - let mut pairs = chunks8.chunks_exact(2); + let (pairs, trailing_chunks) = chunks8.as_chunks::<2>(); - for pair in pairs.by_ref() { + for pair in pairs { let a = u64::from_le_bytes(pair[0]) ^ crc; let b = u64::from_le_bytes(pair[1]); @@ -448,7 +446,7 @@ fn slice16_64_scalar(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u6 } // Handle an odd 8-byte tail - if let [chunk] = pairs.remainder() { + if let [chunk] = trailing_chunks { let val = u64::from_le_bytes(*chunk) ^ crc; crc = tables[7][(val & 0xFF) as usize] ^ tables[6][((val >> 8) & 0xFF) as usize] @@ -475,7 +473,7 @@ fn slice16_64_scalar(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u6 /// * `tables` - 16 lookup tables (256 entries each) #[cfg(all(feature = "crc64", not(all(target_arch = "wasm32", target_feature = "simd128"))))] #[inline] -pub fn slice16_64(crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { +pub(in crate::checksum) fn slice16_64(crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { slice16_64_scalar(crc, data, tables) } @@ -485,7 +483,7 @@ pub fn slice16_64(crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { /// loads when `target_feature = "simd128"` is enabled. #[cfg(all(feature = "crc64", target_arch = "wasm32", target_feature = "simd128"))] #[inline] -pub fn slice16_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { +pub(in crate::checksum) fn slice16_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { use core::arch::wasm32::v128; let mut ptr = data.as_ptr(); @@ -493,7 +491,7 @@ pub fn slice16_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { while len >= 16 { // SAFETY: `ptr` is within `data` and `len >= 16`. - let v = unsafe { core::ptr::read_unaligned(ptr as *const v128) }; + let v = unsafe { core::ptr::read_unaligned(ptr.cast::()) }; // Extract lanes using WASM SIMD intrinsics (avoids transmute layout assumptions). let a = core::arch::wasm32::u64x2_extract_lane::<0>(v) ^ crc; let b = core::arch::wasm32::u64x2_extract_lane::<1>(v); @@ -530,45 +528,12 @@ pub fn slice16_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { #[cfg(test)] mod tests { use super::*; - - // CRC-16 Tests - - /// Generate CRC-16 tables for testing (CCITT polynomial 0x8408 reflected). #[cfg(feature = "crc16")] - const fn crc16_table_entry(poly: u16, index: u8) -> u16 { - let mut crc = index as u16; - let mut i: u32 = 0; - while i < 8 { - if crc & 1 != 0 { - crc = (crc >> 1) ^ poly; - } else { - crc >>= 1; - } - i = i.strict_add(1); - } - crc - } + use crate::checksum::common::tables::{CRC16_CCITT_POLY, generate_crc16_tables_8}; + #[cfg(feature = "crc64")] + use crate::checksum::common::tables::{CRC64_XZ_POLY, generate_crc64_tables_8, generate_crc64_tables_16}; - #[cfg(feature = "crc16")] - const fn generate_crc16_tables_8(poly: u16) -> [[u16; 256]; 8] { - let mut tables = [[0u16; 256]; 8]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc16_table_entry(poly, i as u8); - i = i.strict_add(1); - } - let mut k = 1usize; - while k < 8 { - i = 0; - while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); - i = i.strict_add(1); - } - k = k.strict_add(1); - } - tables - } + // CRC-16 Tests // CRC-32 Tests @@ -600,9 +565,6 @@ mod tests { assert_eq!(slice8_32(!0, data, &tables8), slice16_32(!0, data, &tables16)); } - #[cfg(feature = "crc16")] - const CRC16_CCITT_POLY: u16 = 0x8408; // Reflected - #[test] #[cfg(feature = "crc16")] fn test_slice8_16_empty() { @@ -634,67 +596,6 @@ mod tests { // CRC-64 Tests - /// Generate CRC-64 tables for testing. - #[cfg(feature = "crc64")] - const fn crc64_table_entry(poly: u64, index: u8) -> u64 { - let mut crc = index as u64; - let mut i: u32 = 0; - while i < 8 { - if crc & 1 != 0 { - crc = (crc >> 1) ^ poly; - } else { - crc >>= 1; - } - i = i.strict_add(1); - } - crc - } - - #[cfg(feature = "crc64")] - const fn generate_crc64_tables_8(poly: u64) -> [[u64; 256]; 8] { - let mut tables = [[0u64; 256]; 8]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc64_table_entry(poly, i as u8); - i = i.strict_add(1); - } - let mut k = 1usize; - while k < 8 { - i = 0; - while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); - i = i.strict_add(1); - } - k = k.strict_add(1); - } - tables - } - - #[cfg(feature = "crc64")] - const fn generate_crc64_tables_16(poly: u64) -> [[u64; 256]; 16] { - let mut tables = [[0u64; 256]; 16]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc64_table_entry(poly, i as u8); - i = i.strict_add(1); - } - let mut k = 1usize; - while k < 16 { - i = 0; - while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); - i = i.strict_add(1); - } - k = k.strict_add(1); - } - tables - } - - #[cfg(feature = "crc64")] - const CRC64_XZ_POLY: u64 = 0xC96C_5795_D787_0F42; // Reflected - #[test] #[cfg(feature = "crc64")] fn test_slice8_64_empty() { diff --git a/src/checksum/common/prefetch.rs b/src/checksum/common/prefetch.rs index ea4dba90..ca781e0b 100644 --- a/src/checksum/common/prefetch.rs +++ b/src/checksum/common/prefetch.rs @@ -3,15 +3,6 @@ // SAFETY: This module provides low-level prefetch intrinsics that require unsafe. // Prefetch instructions are hints to the CPU and cannot cause memory unsafety; // invalid addresses are silently ignored. -#![allow(unsafe_code)] -#![cfg_attr( - all( - target_arch = "aarch64", - feature = "crc32", - not(any(feature = "crc16", feature = "crc24", feature = "crc64", test)) - ), - allow(dead_code, unused_imports) -)] //! This module provides architecture-specific prefetch distances and inline //! helpers for large-buffer CRC computation. //! @@ -27,18 +18,20 @@ //! ```text //! use crate::checksum::common::prefetch::{prefetch_read_l1, LARGE_BLOCK_DISTANCE}; //! -//! // In a double-unrolled loop processing 512B per iteration: -//! while ptr.add(DOUBLE_BLOCK) <= end { -//! // Prefetch 2 iterations ahead (1KB for 512B blocks) -//! prefetch_read_l1(ptr.add(LARGE_BLOCK_DISTANCE)); +//! if end.offset_from_unsigned(ptr) >= DOUBLE_BLOCK { +//! let last_double_block = end.sub(DOUBLE_BLOCK); +//! while ptr <= last_double_block { +//! // Prefetch accepts arbitrary addresses, so use wrapping addition. +//! prefetch_read_l1(ptr.wrapping_add(LARGE_BLOCK_DISTANCE)); //! -//! // Process first 256B block -//! // ... fold operations ... +//! // Process first 256B block +//! // ... fold operations ... //! -//! // Process second 256B block -//! // ... fold operations ... +//! // Process second 256B block +//! // ... fold operations ... //! -//! ptr = ptr.add(DOUBLE_BLOCK); +//! ptr = ptr.add(DOUBLE_BLOCK); +//! } //! } //! ``` @@ -48,13 +41,13 @@ /// /// The x86-64 folding loops use this 1,024-byte lookahead. #[cfg(target_arch = "x86_64")] -pub const LARGE_BLOCK_DISTANCE: usize = 1024; +pub(in crate::checksum) const LARGE_BLOCK_DISTANCE: usize = 1024; /// Prefetch distance for large buffer kernels on ARM64. /// /// The AArch64 folding loops use this 768-byte lookahead. #[cfg(target_arch = "aarch64")] -pub const LARGE_BLOCK_DISTANCE: usize = 768; +pub(in crate::checksum) const LARGE_BLOCK_DISTANCE: usize = 768; // x86-64 Prefetch Intrinsics @@ -72,7 +65,7 @@ mod x86_64_impl { /// The pointer does not need to be valid or aligned. Prefetch is a hint; /// invalid addresses are silently ignored by the CPU. #[inline(always)] - pub(crate) unsafe fn prefetch_read_l1(ptr: *const u8) { + pub(in crate::checksum) unsafe fn prefetch_read_l1(ptr: *const u8) { // SAFETY: Prefetch is a CPU hint; invalid addresses are silently ignored. // The _mm_prefetch intrinsic cannot cause memory unsafety. unsafe { @@ -101,7 +94,7 @@ mod aarch64_impl { /// The pointer does not need to be valid or aligned. Prefetch is a hint; /// invalid addresses are silently ignored by the CPU. #[inline(always)] - pub(crate) unsafe fn prefetch_read_l1(ptr: *const u8) { + pub(in crate::checksum) unsafe fn prefetch_read_l1(ptr: *const u8) { // SAFETY: Inline assembly for PRFM prefetch hint. Prefetch instructions // are CPU hints that cannot cause memory unsafety; invalid addresses // are silently ignored by the hardware. @@ -120,13 +113,13 @@ mod aarch64_impl { // Public API #[cfg(target_arch = "aarch64")] -pub(crate) use aarch64_impl::prefetch_read_l1; +pub(in crate::checksum) use aarch64_impl::prefetch_read_l1; #[cfg(target_arch = "x86_64")] -pub(crate) use x86_64_impl::prefetch_read_l1; +pub(in crate::checksum) use x86_64_impl::prefetch_read_l1; // Fallback for other architectures (no-op) #[cfg(all(not(any(target_arch = "x86_64", target_arch = "aarch64")), test))] -pub const LARGE_BLOCK_DISTANCE: usize = 512; +pub(in crate::checksum) const LARGE_BLOCK_DISTANCE: usize = 512; #[cfg(all(not(any(target_arch = "x86_64", target_arch = "aarch64")), test))] #[inline(always)] @@ -135,7 +128,7 @@ pub const LARGE_BLOCK_DISTANCE: usize = 512; /// # Safety /// /// This function performs no memory access and is always safe to call. -pub(crate) unsafe fn prefetch_read_l1(_ptr: *const u8) {} +pub(in crate::checksum) unsafe fn prefetch_read_l1(_ptr: *const u8) {} // Tests diff --git a/src/checksum/common/reference.rs b/src/checksum/common/reference.rs index 92826a08..261ac18f 100644 --- a/src/checksum/common/reference.rs +++ b/src/checksum/common/reference.rs @@ -35,7 +35,6 @@ // SAFETY: All array indexing uses bounded loop indices (0..data.len()). // Clippy cannot prove this in const fn contexts, but bounds are statically guaranteed. -#![allow(clippy::indexing_slicing)] // CRC-16 Reference Implementation @@ -55,7 +54,7 @@ /// The raw CRC register state (caller applies final XOR if needed). #[cfg(feature = "crc16")] #[must_use] -pub const fn crc16_bitwise(poly: u16, init: u16, data: &[u8]) -> u16 { +pub(in crate::checksum) const fn crc16_bitwise(poly: u16, init: u16, data: &[u8]) -> u16 { let mut crc = init; let mut i: usize = 0; while i < data.len() { @@ -88,7 +87,7 @@ pub const fn crc16_bitwise(poly: u16, init: u16, data: &[u8]) -> u16 { /// The CRC value in the low 24 bits. #[cfg(feature = "crc24")] #[must_use] -pub const fn crc24_bitwise(poly: u32, init: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) const fn crc24_bitwise(poly: u32, init: u32, data: &[u8]) -> u32 { // Work in expanded form: CRC in top 24 bits of u32 let poly_expanded = poly.strict_shl(8); let mut crc = (init & 0x00FF_FFFF).strict_shl(8); @@ -130,7 +129,7 @@ pub const fn crc24_bitwise(poly: u32, init: u32, data: &[u8]) -> u32 { /// The raw CRC register state (caller applies final XOR if needed). #[cfg(feature = "crc32")] #[must_use] -pub const fn crc32_bitwise(poly: u32, init: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) const fn crc32_bitwise(poly: u32, init: u32, data: &[u8]) -> u32 { let mut crc = init; let mut i: usize = 0; while i < data.len() { @@ -163,7 +162,7 @@ pub const fn crc32_bitwise(poly: u32, init: u32, data: &[u8]) -> u32 { /// The raw CRC register state (caller applies final XOR if needed). #[cfg(feature = "crc64")] #[must_use] -pub const fn crc64_bitwise(poly: u64, init: u64, data: &[u8]) -> u64 { +pub(in crate::checksum) const fn crc64_bitwise(poly: u64, init: u64, data: &[u8]) -> u64 { let mut crc = init; let mut i: usize = 0; while i < data.len() { @@ -397,11 +396,16 @@ mod tests { #[cfg(all(feature = "crc16", feature = "crc24", feature = "crc32", feature = "crc64"))] fn all_widths_handle_large_input() { // Verify all widths can handle larger inputs without panic - let data: [u8; 1024] = core::array::from_fn(|i| (i as u8).wrapping_mul(17)); - - let _ = crc16_bitwise(CRC16_CCITT_POLY, !0u16, &data); - let _ = crc24_bitwise(CRC24_OPENPGP_POLY, 0x00B7_04CE, &data); - let _ = crc32_bitwise(CRC32_IEEE_POLY, !0u32, &data); - let _ = crc64_bitwise(CRC64_XZ_POLY, !0u64, &data); + let mut byte = 0u8; + let data: [u8; 1024] = core::array::from_fn(|_| { + let value = byte.wrapping_mul(17); + byte = byte.wrapping_add(1); + value + }); + + let _crc16 = crc16_bitwise(CRC16_CCITT_POLY, !0u16, &data); + let _crc24 = crc24_bitwise(CRC24_OPENPGP_POLY, 0x00B7_04CE, &data); + let _crc32 = crc32_bitwise(CRC32_IEEE_POLY, !0u32, &data); + let _crc64 = crc64_bitwise(CRC64_XZ_POLY, !0u64, &data); } } diff --git a/src/checksum/common/tables.rs b/src/checksum/common/tables.rs index 581edfd0..f70b6545 100644 --- a/src/checksum/common/tables.rs +++ b/src/checksum/common/tables.rs @@ -19,7 +19,6 @@ // SAFETY: All array indexing in this module uses bounded loop indices (0..256, 0..N). // Clippy cannot prove this in const fn contexts, but bounds are statically guaranteed. -#![allow(clippy::indexing_slicing)] // CRC-16 Table Generation @@ -28,7 +27,7 @@ /// Uses bit-by-bit computation with the reflected polynomial. #[cfg(feature = "crc16")] #[must_use] -pub const fn crc16_table_entry(poly: u16, index: u8) -> u16 { +pub(super) const fn crc16_table_entry(poly: u16, index: u8) -> u16 { let mut crc = index as u16; let mut i: u32 = 0; while i < 8 { @@ -49,21 +48,24 @@ pub const fn crc16_table_entry(poly: u16, index: u8) -> u16 { /// * `poly` - The reflected polynomial #[cfg(feature = "crc16")] #[must_use] -pub const fn generate_crc16_tables_8(poly: u16) -> [[u16; 256]; 8] { +pub(in crate::checksum) const fn generate_crc16_tables_8(poly: u16) -> [[u16; 256]; 8] { let mut tables = [[0u16; 256]; 8]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc16_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + tables[0][index as usize] = crc16_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } let mut k = 1usize; while k < 8 { - i = 0; + let mut i = 0usize; while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); + let prev = tables[k.strict_sub(1)][i]; + tables[k][i] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); i = i.strict_add(1); } k = k.strict_add(1); @@ -84,7 +86,7 @@ pub const fn generate_crc16_tables_8(poly: u16) -> [[u16; 256]; 8] { /// The returned entry has its low 8 bits cleared. #[cfg(feature = "crc24")] #[must_use] -pub const fn crc24_table_entry(poly: u32, index: u8) -> u32 { +const fn crc24_table_entry(poly: u32, index: u8) -> u32 { // Expand a 24-bit CRC into the top 24 bits of a u32. // Polynomial is aligned to the same position (<< 8). let poly = poly.strict_shl(8); @@ -108,21 +110,24 @@ pub const fn crc24_table_entry(poly: u32, index: u8) -> u32 { /// * `poly` - The normal polynomial (low 24 bits), e.g. 0x864CFB for OpenPGP #[cfg(feature = "crc24")] #[must_use] -pub const fn generate_crc24_tables_8(poly: u32) -> [[u32; 256]; 8] { +pub(in crate::checksum) const fn generate_crc24_tables_8(poly: u32) -> [[u32; 256]; 8] { let mut tables = [[0u32; 256]; 8]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc24_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + tables[0][index as usize] = crc24_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } let mut k = 1usize; while k < 8 { - i = 0; + let mut i = 0usize; while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev >> 24) as usize] ^ (prev.strict_shl(8)); + let prev = tables[k.strict_sub(1)][i]; + tables[k][i] = tables[0][(prev >> 24) as usize] ^ (prev.strict_shl(8)); i = i.strict_add(1); } k = k.strict_add(1); @@ -138,7 +143,7 @@ pub const fn generate_crc24_tables_8(poly: u32) -> [[u32; 256]; 8] { /// Uses bit-by-bit computation with the reflected polynomial. #[cfg(feature = "crc32")] #[must_use] -pub const fn crc32_table_entry(poly: u32, index: u8) -> u32 { +const fn crc32_table_entry(poly: u32, index: u8) -> u32 { let mut crc = index as u32; let mut i: u32 = 0; while i < 8 { @@ -159,21 +164,24 @@ pub const fn crc32_table_entry(poly: u32, index: u8) -> u32 { /// * `poly` - The reflected polynomial #[cfg(all(test, feature = "crc32"))] #[must_use] -pub const fn generate_crc32_tables_8(poly: u32) -> [[u32; 256]; 8] { +pub(super) const fn generate_crc32_tables_8(poly: u32) -> [[u32; 256]; 8] { let mut tables = [[0u32; 256]; 8]; - let mut i: u16 = 0; - while i < 256 { - tables[0][i as usize] = crc32_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + tables[0][index as usize] = crc32_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } let mut k: usize = 1; while k < 8 { - i = 0; + let mut i = 0usize; while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); + let prev = tables[k.strict_sub(1)][i]; + tables[k][i] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); i = i.strict_add(1); } k = k.strict_add(1); @@ -189,21 +197,24 @@ pub const fn generate_crc32_tables_8(poly: u32) -> [[u32; 256]; 8] { /// * `poly` - The reflected polynomial #[cfg(feature = "crc32")] #[must_use] -pub const fn generate_crc32_tables_16(poly: u32) -> [[u32; 256]; 16] { +pub(in crate::checksum) const fn generate_crc32_tables_16(poly: u32) -> [[u32; 256]; 16] { let mut tables = [[0u32; 256]; 16]; - let mut i: u16 = 0; - while i < 256 { - tables[0][i as usize] = crc32_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + tables[0][index as usize] = crc32_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } let mut k: usize = 1; while k < 16 { - i = 0; + let mut i = 0usize; while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); + let prev = tables[k.strict_sub(1)][i]; + tables[k][i] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); i = i.strict_add(1); } k = k.strict_add(1); @@ -219,7 +230,7 @@ pub const fn generate_crc32_tables_16(poly: u32) -> [[u32; 256]; 16] { /// Uses bit-by-bit computation with the reflected polynomial. #[cfg(feature = "crc64")] #[must_use] -pub const fn crc64_table_entry(poly: u64, index: u8) -> u64 { +pub(super) const fn crc64_table_entry(poly: u64, index: u8) -> u64 { let mut crc = index as u64; let mut i: u32 = 0; while i < 8 { @@ -240,21 +251,24 @@ pub const fn crc64_table_entry(poly: u64, index: u8) -> u64 { /// * `poly` - The reflected polynomial #[cfg(all(feature = "crc64", any(target_arch = "x86_64", target_arch = "aarch64", test)))] #[must_use] -pub const fn generate_crc64_tables_8(poly: u64) -> [[u64; 256]; 8] { +pub(in crate::checksum) const fn generate_crc64_tables_8(poly: u64) -> [[u64; 256]; 8] { let mut tables = [[0u64; 256]; 8]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc64_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + tables[0][index as usize] = crc64_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } let mut k = 1usize; while k < 8 { - i = 0; + let mut i = 0usize; while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); + let prev = tables[k.strict_sub(1)][i]; + tables[k][i] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); i = i.strict_add(1); } k = k.strict_add(1); @@ -273,21 +287,24 @@ pub const fn generate_crc64_tables_8(poly: u64) -> [[u64; 256]; 8] { /// * `poly` - The reflected polynomial #[cfg(feature = "crc64")] #[must_use] -pub const fn generate_crc64_tables_16(poly: u64) -> [[u64; 256]; 16] { +pub(in crate::checksum) const fn generate_crc64_tables_16(poly: u64) -> [[u64; 256]; 16] { let mut tables = [[0u64; 256]; 16]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc64_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + tables[0][index as usize] = crc64_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } let mut k = 1usize; while k < 16 { - i = 0; + let mut i = 0usize; while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); + let prev = tables[k.strict_sub(1)][i]; + tables[k][i] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); i = i.strict_add(1); } k = k.strict_add(1); @@ -303,19 +320,19 @@ pub const fn generate_crc64_tables_16(poly: u64) -> [[u64; 256]; 16] { /// CRC-16-CCITT polynomial (0x1021) in reflected form. /// Used by X.25, V.41, HDLC, XMODEM, Bluetooth, PACTOR, SD, etc. #[cfg(feature = "crc16")] -pub const CRC16_CCITT_POLY: u16 = 0x8408; +pub(in crate::checksum) const CRC16_CCITT_POLY: u16 = 0x8408; /// CRC-16-IBM polynomial (0x8005) in reflected form. /// Used by Modbus, USB, ANSI X3.28, etc. #[cfg(feature = "crc16")] -pub const CRC16_IBM_POLY: u16 = 0xA001; +pub(in crate::checksum) const CRC16_IBM_POLY: u16 = 0xA001; // CRC-24 Polynomials /// CRC-24-OPENPGP polynomial (0x864CFB) in normal form. /// Used by OpenPGP (RFC 4880). #[cfg(feature = "crc24")] -pub const CRC24_OPENPGP_POLY: u32 = 0x0086_4CFB; +pub(in crate::checksum) const CRC24_OPENPGP_POLY: u32 = 0x0086_4CFB; // CRC-64 Polynomials @@ -325,23 +342,23 @@ pub const CRC24_OPENPGP_POLY: u32 = 0x0086_4CFB; /// /// Used by Ethernet, gzip, zip, PNG, etc. #[cfg(feature = "crc32")] -pub const CRC32_IEEE_POLY: u32 = 0xEDB8_8320; +pub(in crate::checksum) const CRC32_IEEE_POLY: u32 = 0xEDB8_8320; /// CRC-32C (Castagnoli) polynomial (0x1EDC6F41) in reflected form. /// /// Used by iSCSI, SCTP, ext4, Btrfs, SSE4.2 `crc32`, etc. #[cfg(feature = "crc32")] -pub const CRC32C_POLY: u32 = 0x82F6_3B78; +pub(in crate::checksum) const CRC32C_POLY: u32 = 0x82F6_3B78; /// CRC-64-XZ polynomial (0x42F0E1EBA9EA3693) in reflected form. /// Used by XZ Utils, 7-Zip, LZMA. #[cfg(feature = "crc64")] -pub const CRC64_XZ_POLY: u64 = 0xC96C_5795_D787_0F42; +pub(in crate::checksum) const CRC64_XZ_POLY: u64 = 0xC96C_5795_D787_0F42; /// CRC-64-NVME polynomial (0xAD93D23594C93659) in reflected form. /// Used by NVMe specification. #[cfg(feature = "crc64")] -pub const CRC64_NVME_POLY: u64 = 0x9A6C_9329_AC4B_C9B5; +pub(in crate::checksum) const CRC64_NVME_POLY: u64 = 0x9A6C_9329_AC4B_C9B5; // Tests diff --git a/src/checksum/common/tests.rs b/src/checksum/common/tests.rs index a4f32683..06c86df8 100644 --- a/src/checksum/common/tests.rs +++ b/src/checksum/common/tests.rs @@ -13,15 +13,13 @@ //! //! Used via [`define_crc_property_tests!`] macro in algorithm modules. -#![cfg_attr(miri, allow(dead_code))] - use crate::traits::{Checksum, ChecksumCombine}; /// Generic test harness for CRC algorithms. /// /// Provides a suite of property-based tests that verify fundamental CRC invariants. /// These tests can be run against any type implementing `Checksum + ChecksumCombine`. -pub struct CrcTestHarness { +pub(in crate::checksum) struct CrcTestHarness { _phantom: core::marker::PhantomData, } @@ -36,8 +34,8 @@ where /// /// This is the fundamental combine property that enables parallel CRC computation. #[inline] - pub fn test_combine_property(data: &[u8], split: usize) { - let split = if data.is_empty() { 0 } else { split % data.len() }; + pub(in crate::checksum) fn test_combine_property(data: &[u8], split: usize) { + let split = split.checked_rem(data.len()).unwrap_or(0); let (a, b) = data.split_at(split); let crc_a = C::checksum(a); @@ -49,9 +47,9 @@ where } /// Test combine property at all possible split points for a given buffer. + #[cfg(feature = "crc64")] #[inline] - #[cfg_attr(test, allow(dead_code))] - pub fn test_combine_all_splits(data: &[u8]) { + pub(in crate::checksum) fn test_combine_all_splits(data: &[u8]) { let full = C::checksum(data); for split in 0..=data.len() { @@ -66,7 +64,7 @@ where /// Test combine with empty second part (identity case). #[inline] - pub fn test_combine_empty_suffix(data: &[u8]) { + pub(in crate::checksum) fn test_combine_empty_suffix(data: &[u8]) { let crc_data = C::checksum(data); let crc_empty = C::checksum(&[]); let combined = C::combine(crc_data, crc_empty, 0); @@ -76,7 +74,7 @@ where /// Test combine with empty first part. #[inline] - pub fn test_combine_empty_prefix(data: &[u8]) { + pub(in crate::checksum) fn test_combine_empty_prefix(data: &[u8]) { let crc_empty = C::checksum(&[]); let crc_data = C::checksum(data); let combined = C::combine(crc_empty, crc_data, data.len()); @@ -87,9 +85,9 @@ where // Streaming Consistency Tests /// Test that streaming updates produce the same result as one-shot. + #[cfg(feature = "crc64")] #[inline] - #[cfg_attr(test, allow(dead_code))] - pub fn test_streaming_equals_oneshot(data: &[u8]) { + pub(in crate::checksum) fn test_streaming_equals_oneshot(data: &[u8]) { let oneshot = C::checksum(data); let mut hasher = C::new(); @@ -101,7 +99,7 @@ where /// Test streaming with byte-at-a-time updates. #[inline] - pub fn test_streaming_byte_at_a_time(data: &[u8]) { + pub(in crate::checksum) fn test_streaming_byte_at_a_time(data: &[u8]) { let oneshot = C::checksum(data); let mut hasher = C::new(); @@ -115,7 +113,7 @@ where /// Test streaming across a specific chunk size boundary. #[inline] - pub fn test_streaming_chunked(data: &[u8], chunk_size: usize) { + pub(in crate::checksum) fn test_streaming_chunked(data: &[u8], chunk_size: usize) { if chunk_size == 0 { return; } @@ -133,7 +131,7 @@ where /// Test that finalize is idempotent (can be called multiple times). #[inline] - pub fn test_finalize_idempotent(data: &[u8]) { + pub(in crate::checksum) fn test_finalize_idempotent(data: &[u8]) { let mut hasher = C::new(); hasher.update(data); @@ -149,7 +147,7 @@ where /// Test that reset returns the hasher to its initial state. #[inline] - pub fn test_reset(data: &[u8]) { + pub(in crate::checksum) fn test_reset(data: &[u8]) { let fresh = C::checksum(data); let mut hasher = C::new(); @@ -170,7 +168,7 @@ where /// 2. Computing second half separately /// 3. Combining the results #[inline] - pub fn test_streaming_and_combine(data: &[u8]) { + pub(in crate::checksum) fn test_streaming_and_combine(data: &[u8]) { if data.is_empty() { return; } @@ -197,7 +195,7 @@ where /// Test empty input. #[inline] - pub fn test_empty_input() { + pub(in crate::checksum) fn test_empty_input() { let oneshot = C::checksum(&[]); let hasher = C::new(); @@ -208,7 +206,7 @@ where /// Test single byte inputs for all byte values. #[inline] - pub fn test_single_bytes() { + pub(in crate::checksum) fn test_single_bytes() { for byte in 0u8..=255 { let oneshot = C::checksum(&[byte]); @@ -302,7 +300,7 @@ macro_rules! define_crc_property_tests { /// Used by all CRC cross-check test modules to exercise edge cases around /// lane widths, cache line boundaries, and page boundaries. #[cfg(not(miri))] -pub const TEST_LENGTHS: &[usize] = &[ +pub(in crate::checksum) const TEST_LENGTHS: &[usize] = &[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // Tiny 16, 17, 31, 32, 33, 63, 64, 65, // SSE/NEON boundaries 127, 128, 129, 255, 256, 257, // Cache line boundaries @@ -311,25 +309,23 @@ pub const TEST_LENGTHS: &[usize] = &[ 8192, 16384, 32768, 65536, // Large buffers ]; #[cfg(miri)] -pub const TEST_LENGTHS: &[usize] = &[ +pub(in crate::checksum) const TEST_LENGTHS: &[usize] = &[ 0, 1, 2, 3, 7, 8, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, 256, 257, 511, 512, 513, 1023, 1024, 1025, ]; /// Prime-sized chunk patterns for streaming cross-check tests. #[cfg(not(miri))] -pub const STREAMING_CHUNK_SIZES: &[usize] = &[1, 3, 7, 13, 17, 31, 37, 61, 127, 251]; +pub(in crate::checksum) const STREAMING_CHUNK_SIZES: &[usize] = &[1, 3, 7, 13, 17, 31, 37, 61, 127, 251]; #[cfg(miri)] -pub const STREAMING_CHUNK_SIZES: &[usize] = &[1, 3, 7, 13, 31, 61]; +pub(in crate::checksum) const STREAMING_CHUNK_SIZES: &[usize] = &[1, 3, 7, 13, 31, 61]; /// Generate deterministic test data of a given length. /// /// Uses a simple mixing function to produce non-trivial byte patterns /// that avoid accidentally passing due to regularity. -#[cfg_attr(test, allow(dead_code))] -pub fn generate_test_data(len: usize) -> alloc::vec::Vec { - (0..len) - .map(|i| (i as u64).wrapping_mul(17).wrapping_add(i as u64) as u8) - .collect() +#[cfg(any(feature = "crc16", feature = "crc24"))] +pub(in crate::checksum) fn generate_test_data(len: usize) -> alloc::vec::Vec { + (0u8..=u8::MAX).cycle().take(len).map(|i| i.wrapping_mul(18)).collect() } // Tests for the test harness itself diff --git a/src/checksum/crc16/aarch64.rs b/src/checksum/crc16/aarch64.rs index 64ce2ed1..4c185675 100644 --- a/src/checksum/crc16/aarch64.rs +++ b/src/checksum/crc16/aarch64.rs @@ -6,10 +6,8 @@ //! //! # Safety //! -//! Uses `unsafe` for ARM SIMD intrinsics. Callers must ensure PMULL is -//! available before executing these kernels (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] +//! Uses `unsafe` for ARM SIMD intrinsics. Callers must establish NEON, AES +//! (PMULL), and SHA3 (EOR3) support as required by each accelerated path. use core::{ arch::aarch64::*, @@ -42,33 +40,57 @@ impl BitXorAssign for Simd { } impl Simd { + /// Creates a vector from its high and low 64-bit lanes. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn new(high: u64, low: u64) -> Self { Self(vcombine_u8(vcreate_u8(low), vcreate_u8(high))) } + /// Loads 16 bytes from `ptr` without requiring alignment. + /// + /// # Safety + /// + /// The current CPU must support NEON, and `ptr` must address at least 16 initialized readable + /// bytes. #[inline] #[target_feature(enable = "neon")] unsafe fn load(ptr: *const u8) -> Self { - // SAFETY: Caller guarantees: - // 1. NEON target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + // SAFETY: The caller guarantees NEON support and 16 initialized readable bytes at `ptr`. unsafe { Self(vld1q_u8(ptr)) } } + /// Computes the bitwise AND of two vectors. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn and(self, mask: Self) -> Self { Self(vandq_u8(self.0, mask.0)) } + /// Shifts the vector right by eight bytes, filling the high bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn shift_right_8(self) -> Self { Self(vextq_u8(self.0, vdupq_n_u8(0), 8)) } + /// Moves the low 32-bit lane to the high 32-bit lane and clears the rest. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn shift_left_12(self) -> Self { @@ -77,6 +99,11 @@ impl Simd { Self(vreinterpretq_u8_u32(result)) } + /// Multiplies the low polynomial lanes. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul00(self, other: Self) -> Self { @@ -85,6 +112,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Multiplies this vector's high polynomial lane by the other vector's low lane. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul01(self, other: Self) -> Self { @@ -93,6 +125,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Multiplies this vector's low polynomial lane by the other vector's high lane. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul10(self, other: Self) -> Self { @@ -101,6 +138,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Multiplies the high polynomial lanes. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul11(self, other: Self) -> Self { @@ -109,6 +151,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Folds one reflected 16-byte lane and XORs the supplied input lane. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { @@ -122,6 +169,11 @@ impl Simd { } } + /// Folds a reflected CRC state from 128 bits to the width-32 reduction state. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn fold_width32_reflected(self, high: u64, low: u64) -> Self { @@ -146,9 +198,14 @@ impl Simd { } } + /// Applies Barrett reduction and returns the low CRC-16 state. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] - unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { + unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u16 { // SAFETY: Caller guarantees: // 1. AES + NEON target features are available (dispatch check). // 2. All SIMD operations are pure register computations after loads. @@ -159,14 +216,19 @@ impl Simd { let xorred = self ^ clmul2; let hi = xorred.shift_right_8(); - vgetq_lane_u32(vreinterpretq_u32_u8(hi.0), 0) + vgetq_lane_u16(vreinterpretq_u16_u8(hi.0), 0) } } } +/// Reduces eight folded SIMD lanes to a CRC-16 state. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] -unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u32 { +unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u16 { // SAFETY: Caller guarantees: // 1. AES + NEON target features are available (dispatch check). // 2. All SIMD operations are pure register computations after loads. @@ -185,6 +247,11 @@ unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u3 } } +/// Folds one reflected 128-byte block into the current SIMD state. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn fold_block_128_width32_reflected(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { @@ -203,8 +270,17 @@ unsafe fn fold_block_128_width32_reflected(x: &mut [Simd; 8], chunk: &[Simd; 8], } } +/// Folds one reflected 16-byte lane with a three-input XOR. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn fold_lane_width32_reflected_eor3(x: Simd, coeff: Simd, data_to_xor: Simd) -> Simd { // SAFETY: Caller guarantees: @@ -217,8 +293,17 @@ unsafe fn fold_lane_width32_reflected_eor3(x: Simd, coeff: Simd, data_to_xor: Si } } +/// Folds one reflected 128-byte block with EOR3 into the current SIMD state. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn fold_block_128_width32_reflected_eor3(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { // SAFETY: Caller guarantees: @@ -236,13 +321,18 @@ unsafe fn fold_block_128_width32_reflected_eor3(x: &mut [Simd; 8], chunk: &[Simd } } +/// Folds reflected 128-byte blocks through two independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_width32_reflected_2way( - state: u32, + state: u16, blocks: &[[Simd; 8]], fold_256b: (u64, u64), keys: &[u64; 23], -) -> u32 { +) -> u16 { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; // SAFETY: Caller guarantees: @@ -250,10 +340,11 @@ unsafe fn update_simd_width32_reflected_2way( // 2. All pointer arithmetic stays within bounds via loop guards. // 3. All SIMD operations are pure register computations after loads. unsafe { - debug_assert!(blocks.len() >= 2); if blocks.len() < 2 { - // SAFETY: this function is only called when there are at least 2 blocks. - core::hint::unreachable_unchecked() + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_width32_reflected(state, first, rest, keys); } let coeff_256b = Simd::new(fold_256b.0, fold_256b.1); @@ -267,7 +358,7 @@ unsafe fn update_simd_width32_reflected_2way( let mut s1 = *blocks_ptr.add(1); // Inject CRC into stream 0 (block 0). - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, u64::from(state)); // Pointer-walk variant to keep the hot loop free of index bounds checks. const BLOCK_SIZE: usize = 128; @@ -276,7 +367,7 @@ unsafe fn update_simd_width32_reflected_2way( let blocks_end = blocks_ptr.add(blocks.len()); let mut ptr = blocks_ptr.add(2); - let mut rem = blocks.len() - 2; + let mut rem = blocks.len().strict_sub(2); while rem >= DOUBLE_GROUP { let prefetch_ptr = ptr.add(PREFETCH_BLOCKS); @@ -293,7 +384,7 @@ unsafe fn update_simd_width32_reflected_2way( fold_block_128_width32_reflected(&mut s1, &*ptr.add(3), coeff_256b); ptr = ptr.add(DOUBLE_GROUP); - rem -= DOUBLE_GROUP; + rem = rem.strict_sub(DOUBLE_GROUP); } // Handle remaining pairs. @@ -301,7 +392,7 @@ unsafe fn update_simd_width32_reflected_2way( fold_block_128_width32_reflected(&mut s0, &*ptr, coeff_256b); fold_block_128_width32_reflected(&mut s1, &*ptr.add(1), coeff_256b); ptr = ptr.add(2); - rem -= 2; + rem = rem.strict_sub(2); } // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). @@ -324,14 +415,19 @@ unsafe fn update_simd_width32_reflected_2way( } } +/// Folds reflected 128-byte blocks through three independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_width32_reflected_3way( - state: u32, + state: u16, blocks: &[[Simd; 8]], fold_384b: (u64, u64), fold_256b: (u64, u64), keys: &[u64; 23], -) -> u32 { +) -> u16 { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; // SAFETY: Caller guarantees: @@ -356,7 +452,7 @@ unsafe fn update_simd_width32_reflected_3way( let mut s2 = blocks[2]; // Inject CRC into stream 0 (block 0). - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, u64::from(state)); // Double-unrolled main loop: process 6 blocks (768B) per iteration. const BLOCK_SIZE: usize = 128; @@ -366,7 +462,12 @@ unsafe fn update_simd_width32_reflected_3way( let blocks_ptr = blocks.as_ptr(); let blocks_end = blocks_ptr.add(blocks.len()); let mut ptr = blocks_ptr.add(3); - let double_end = blocks_ptr.add(3 + ((blocks.len() - 3) / DOUBLE_GROUP) * DOUBLE_GROUP); + let double_blocks = blocks + .len() + .strict_sub(3) + .strict_div(DOUBLE_GROUP) + .strict_mul(DOUBLE_GROUP); + let double_end = blocks_ptr.add(3usize.strict_add(double_blocks)); while ptr < double_end { let prefetch_ptr = ptr.add(PREFETCH_BLOCKS); @@ -388,7 +489,7 @@ unsafe fn update_simd_width32_reflected_3way( } // Handle remaining triplets. - let triple_end = blocks_ptr.add((blocks.len() / 3) * 3); + let triple_end = blocks_ptr.add(blocks.len().strict_div(3).strict_mul(3)); while ptr < triple_end { fold_block_128_width32_reflected(&mut s0, &*ptr, coeff_384b); fold_block_128_width32_reflected(&mut s1, &*ptr.add(1), coeff_384b); @@ -427,9 +528,14 @@ unsafe fn update_simd_width32_reflected_3way( } } +/// Folds a reflected sequence of 128-byte blocks and reduces it to CRC-16. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] -unsafe fn update_simd_width32_reflected(state: u32, first: &[Simd; 8], rest: &[[Simd; 8]], keys: &[u64; 23]) -> u32 { +unsafe fn update_simd_width32_reflected(state: u16, first: &[Simd; 8], rest: &[[Simd; 8]], keys: &[u64; 23]) -> u16 { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; // SAFETY: Caller guarantees: @@ -439,7 +545,7 @@ unsafe fn update_simd_width32_reflected(state: u32, first: &[Simd; 8], rest: &[[ unsafe { let mut x = *first; - x[0] ^= Simd::new(0, state as u64); + x[0] ^= Simd::new(0, u64::from(state)); let coeff_128b = Simd::new(keys[4], keys[3]); @@ -451,7 +557,7 @@ unsafe fn update_simd_width32_reflected(state: u32, first: &[Simd; 8], rest: &[[ let rest_ptr = rest.as_ptr(); let rest_end = rest_ptr.add(rest.len()); let mut ptr = rest_ptr; - let double_end = rest_ptr.add((rest.len() / DOUBLE_GROUP) * DOUBLE_GROUP); + let double_end = rest_ptr.add(rest.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP)); while ptr < double_end { let prefetch_ptr = ptr.add(PREFETCH_BLOCKS); @@ -499,15 +605,24 @@ unsafe fn update_simd_width32_reflected(state: u32, first: &[Simd; 8], rest: &[[ } } +/// Folds a reflected sequence of 128-byte blocks with EOR3 and reduces it to CRC-16. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_width32_reflected_eor3( - state: u32, + state: u16, first: &[Simd; 8], rest: &[[Simd; 8]], keys: &[u64; 23], -) -> u32 { +) -> u16 { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; // SAFETY: Caller guarantees: @@ -517,7 +632,7 @@ unsafe fn update_simd_width32_reflected_eor3( unsafe { let mut x = *first; - x[0] ^= Simd::new(0, state as u64); + x[0] ^= Simd::new(0, u64::from(state)); let coeff_128b = Simd::new(keys[4], keys[3]); @@ -528,7 +643,7 @@ unsafe fn update_simd_width32_reflected_eor3( let rest_ptr = rest.as_ptr(); let rest_end = rest_ptr.add(rest.len()); let mut ptr = rest_ptr; - let double_end = rest_ptr.add((rest.len() / DOUBLE_GROUP) * DOUBLE_GROUP); + let double_end = rest_ptr.add(rest.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP)); while ptr < double_end { let prefetch_ptr = ptr.add(PREFETCH_BLOCKS); @@ -550,14 +665,23 @@ unsafe fn update_simd_width32_reflected_eor3( } } -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +/// Folds reflected 128-byte blocks through two EOR3-assisted PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_width32_reflected_eor3_2way( - state: u32, + state: u16, blocks: &[[Simd; 8]], fold_256b: (u64, u64), keys: &[u64; 23], -) -> u32 { +) -> u16 { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; // SAFETY: Caller guarantees: @@ -565,10 +689,11 @@ unsafe fn update_simd_width32_reflected_eor3_2way( // 2. All pointer arithmetic stays within bounds via loop guards. // 3. All SIMD operations are pure register computations after loads. unsafe { - debug_assert!(blocks.len() >= 2); if blocks.len() < 2 { - // SAFETY: this function is only called when there are at least 2 blocks. - core::hint::unreachable_unchecked() + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_width32_reflected_eor3(state, first, rest, keys); } let coeff_256b = Simd::new(fold_256b.0, fold_256b.1); @@ -578,7 +703,7 @@ unsafe fn update_simd_width32_reflected_eor3_2way( let mut s0 = *blocks_ptr; let mut s1 = *blocks_ptr.add(1); - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, u64::from(state)); const BLOCK_SIZE: usize = 128; const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B @@ -586,7 +711,7 @@ unsafe fn update_simd_width32_reflected_eor3_2way( let blocks_end = blocks_ptr.add(blocks.len()); let mut ptr = blocks_ptr.add(2); - let mut rem = blocks.len() - 2; + let mut rem = blocks.len().strict_sub(2); while rem >= DOUBLE_GROUP { let prefetch_ptr = ptr.add(PREFETCH_BLOCKS); @@ -600,14 +725,14 @@ unsafe fn update_simd_width32_reflected_eor3_2way( fold_block_128_width32_reflected_eor3(&mut s1, &*ptr.add(3), coeff_256b); ptr = ptr.add(DOUBLE_GROUP); - rem -= DOUBLE_GROUP; + rem = rem.strict_sub(DOUBLE_GROUP); } while rem >= 2 { fold_block_128_width32_reflected_eor3(&mut s0, &*ptr, coeff_256b); fold_block_128_width32_reflected_eor3(&mut s1, &*ptr.add(1), coeff_256b); ptr = ptr.add(2); - rem -= 2; + rem = rem.strict_sub(2); } let mut combined = s1; @@ -628,15 +753,24 @@ unsafe fn update_simd_width32_reflected_eor3_2way( } } -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +/// Folds reflected 128-byte blocks through three EOR3-assisted PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_width32_reflected_eor3_3way( - state: u32, + state: u16, blocks: &[[Simd; 8]], fold_384b: (u64, u64), fold_256b: (u64, u64), keys: &[u64; 23], -) -> u32 { +) -> u16 { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; // SAFETY: Caller guarantees: @@ -659,7 +793,7 @@ unsafe fn update_simd_width32_reflected_eor3_3way( let mut s1 = blocks[1]; let mut s2 = blocks[2]; - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, u64::from(state)); const BLOCK_SIZE: usize = 128; const DOUBLE_GROUP: usize = 6; // 2 × 3-way = 6 blocks = 768B @@ -668,7 +802,12 @@ unsafe fn update_simd_width32_reflected_eor3_3way( let blocks_ptr = blocks.as_ptr(); let blocks_end = blocks_ptr.add(blocks.len()); let mut ptr = blocks_ptr.add(3); - let double_end = blocks_ptr.add(3 + ((blocks.len() - 3) / DOUBLE_GROUP) * DOUBLE_GROUP); + let double_blocks = blocks + .len() + .strict_sub(3) + .strict_div(DOUBLE_GROUP) + .strict_mul(DOUBLE_GROUP); + let double_end = blocks_ptr.add(3usize.strict_add(double_blocks)); while ptr < double_end { let prefetch_ptr = ptr.add(PREFETCH_BLOCKS); @@ -686,7 +825,7 @@ unsafe fn update_simd_width32_reflected_eor3_3way( ptr = ptr.add(DOUBLE_GROUP); } - let triple_end = blocks_ptr.add((blocks.len() / 3) * 3); + let triple_end = blocks_ptr.add(blocks.len().strict_div(3).strict_mul(3)); while ptr < triple_end { fold_block_128_width32_reflected_eor3(&mut s0, &*ptr, coeff_384b); fold_block_128_width32_reflected_eor3(&mut s1, &*ptr.add(1), coeff_384b); @@ -722,6 +861,11 @@ unsafe fn update_simd_width32_reflected_eor3_3way( } } +/// Computes CRC-16 with single-lane PMULL folding for small buffers. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc16_width32_pmull_small( @@ -745,7 +889,7 @@ unsafe fn crc16_width32_pmull_small( let coeff_16b = Simd::new(keys[2], keys[1]); let mut x0 = Simd::load(buf); - x0 ^= Simd::new(0, state as u64); + x0 ^= Simd::new(0, u64::from(state)); buf = buf.add(16); len = len.strict_sub(16); @@ -757,13 +901,18 @@ unsafe fn crc16_width32_pmull_small( } let x0 = x0.fold_width32_reflected(keys[6], keys[5]); - state = x0.barrett_width32_reflected(keys[8], keys[7]) as u16; + state = x0.barrett_width32_reflected(keys[8], keys[7]); let tail = core::slice::from_raw_parts(buf, len); portable(state, tail) } } +/// Computes CRC-16 with PMULL folding. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc16_width32_pmull(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { @@ -777,12 +926,16 @@ unsafe fn crc16_width32_pmull(mut state: u16, data: &[u8], keys: &[u64; 23], por }; state = portable(state, left); - let state32 = update_simd_width32_reflected(state as u32, first, rest, keys); - state = state32 as u16; + state = update_simd_width32_reflected(state, first, rest, keys); portable(state, right) } } +/// Computes CRC-16 with two independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc16_width32_pmull_2way( @@ -802,19 +955,23 @@ unsafe fn crc16_width32_pmull_2way( } state = portable(state, left); - let state32 = if middle.len() >= 2 { - update_simd_width32_reflected_2way(state as u32, middle, fold_256b, keys) + state = if middle.len() >= 2 { + update_simd_width32_reflected_2way(state, middle, fold_256b, keys) } else { let Some((first, rest)) = middle.split_first() else { return crc16_width32_pmull_small(state, data, keys, portable); }; - update_simd_width32_reflected(state as u32, first, rest, keys) + update_simd_width32_reflected(state, first, rest, keys) }; - state = state32 as u16; portable(state, right) } } +/// Computes CRC-16 with three independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc16_width32_pmull_3way( @@ -835,14 +992,22 @@ unsafe fn crc16_width32_pmull_3way( } state = portable(state, left); - let state32 = update_simd_width32_reflected_3way(state as u32, middle, fold_384b, fold_256b, keys); - state = state32 as u16; + state = update_simd_width32_reflected_3way(state, middle, fold_384b, fold_256b, keys); portable(state, right) } } +/// Computes CRC-16 with EOR3-assisted PMULL folding. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn crc16_width32_pmull_eor3( mut state: u16, @@ -860,14 +1025,22 @@ unsafe fn crc16_width32_pmull_eor3( }; state = portable(state, left); - let state32 = update_simd_width32_reflected_eor3(state as u32, first, rest, keys); - state = state32 as u16; + state = update_simd_width32_reflected_eor3(state, first, rest, keys); portable(state, right) } } +/// Computes CRC-16 with two EOR3-assisted PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn crc16_width32_pmull_eor3_2way( mut state: u16, @@ -886,21 +1059,29 @@ unsafe fn crc16_width32_pmull_eor3_2way( } state = portable(state, left); - let state32 = if middle.len() >= 2 { - update_simd_width32_reflected_eor3_2way(state as u32, middle, fold_256b, keys) + state = if middle.len() >= 2 { + update_simd_width32_reflected_eor3_2way(state, middle, fold_256b, keys) } else { let Some((first, rest)) = middle.split_first() else { return crc16_width32_pmull_small(state, data, keys, portable); }; - update_simd_width32_reflected_eor3(state as u32, first, rest, keys) + update_simd_width32_reflected_eor3(state, first, rest, keys) }; - state = state32 as u16; portable(state, right) } } +/// Computes CRC-16 with three EOR3-assisted PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn crc16_width32_pmull_eor3_3way( mut state: u16, @@ -920,22 +1101,17 @@ unsafe fn crc16_width32_pmull_eor3_3way( } state = portable(state, left); - let state32 = update_simd_width32_reflected_eor3_3way(state as u32, middle, fold_384b, fold_256b, keys); - state = state32 as u16; + state = update_simd_width32_reflected_eor3_3way(state, middle, fold_384b, fold_256b, keys); portable(state, right) } } -// Public Safe Kernels (matching CRC-64 pure fn(u16, &[u8]) -> u16 signature) +// Safe kernel wrappers matching the CRC-64 `fn(u16, &[u8]) -> u16` signature. /// CRC-16/CCITT PMULL kernel. -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ccitt_pmull_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ccitt_pmull_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull( crc, @@ -949,13 +1125,9 @@ pub fn crc16_ccitt_pmull_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT PMULL small-buffer kernel. /// /// Optimized for inputs smaller than a folding block (128 bytes). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ccitt_pmull_small_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ccitt_pmull_small_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull_small( crc, @@ -967,13 +1139,9 @@ pub fn crc16_ccitt_pmull_small_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/CCITT PMULL kernel (2-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ccitt_pmull_2way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ccitt_pmull_2way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull_2way( crc, @@ -986,13 +1154,9 @@ pub fn crc16_ccitt_pmull_2way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/CCITT PMULL kernel (3-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ccitt_pmull_3way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ccitt_pmull_3way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull_3way( crc, @@ -1006,14 +1170,14 @@ pub fn crc16_ccitt_pmull_3way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/CCITT PMULL+EOR3 kernel. -/// -/// # Safety -/// -/// Dispatcher verifies PMULL+SHA3 before selecting this kernel. #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] -pub fn crc16_ccitt_pmull_eor3_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL+SHA3 before selecting this kernel. +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] +pub(super) fn crc16_ccitt_pmull_eor3_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL and SHA3 support before invoking this private wrapper. unsafe { crc16_width32_pmull_eor3( crc, @@ -1025,14 +1189,14 @@ pub fn crc16_ccitt_pmull_eor3_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/CCITT PMULL+EOR3 kernel (2-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL+SHA3 before selecting this kernel. #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] -pub fn crc16_ccitt_pmull_eor3_2way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL+SHA3 before selecting this kernel. +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] +pub(super) fn crc16_ccitt_pmull_eor3_2way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL and SHA3 support before invoking this private wrapper. unsafe { crc16_width32_pmull_eor3_2way( crc, @@ -1045,14 +1209,14 @@ pub fn crc16_ccitt_pmull_eor3_2way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/CCITT PMULL+EOR3 kernel (3-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL+SHA3 before selecting this kernel. #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] -pub fn crc16_ccitt_pmull_eor3_3way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL+SHA3 before selecting this kernel. +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] +pub(super) fn crc16_ccitt_pmull_eor3_3way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL and SHA3 support before invoking this private wrapper. unsafe { crc16_width32_pmull_eor3_3way( crc, @@ -1066,37 +1230,25 @@ pub fn crc16_ccitt_pmull_eor3_3way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/IBM PMULL kernel. -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ibm_pmull_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ibm_pmull_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } /// CRC-16/IBM PMULL small-buffer kernel. /// /// Optimized for inputs smaller than a folding block (128 bytes). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ibm_pmull_small_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ibm_pmull_small_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull_small(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } /// CRC-16/IBM PMULL kernel (2-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ibm_pmull_2way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ibm_pmull_2way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull_2way( crc, @@ -1109,13 +1261,9 @@ pub fn crc16_ibm_pmull_2way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/IBM PMULL kernel (3-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ibm_pmull_3way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ibm_pmull_3way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull_3way( crc, @@ -1129,26 +1277,26 @@ pub fn crc16_ibm_pmull_3way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/IBM PMULL+EOR3 kernel. -/// -/// # Safety -/// -/// Dispatcher verifies PMULL+SHA3 before selecting this kernel. #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] -pub fn crc16_ibm_pmull_eor3_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL+SHA3 before selecting this kernel. +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] +pub(super) fn crc16_ibm_pmull_eor3_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL and SHA3 support before invoking this private wrapper. unsafe { crc16_width32_pmull_eor3(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } /// CRC-16/IBM PMULL+EOR3 kernel (2-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL+SHA3 before selecting this kernel. #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] -pub fn crc16_ibm_pmull_eor3_2way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL+SHA3 before selecting this kernel. +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] +pub(super) fn crc16_ibm_pmull_eor3_2way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL and SHA3 support before invoking this private wrapper. unsafe { crc16_width32_pmull_eor3_2way( crc, @@ -1161,14 +1309,14 @@ pub fn crc16_ibm_pmull_eor3_2way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/IBM PMULL+EOR3 kernel (3-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL+SHA3 before selecting this kernel. #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] -pub fn crc16_ibm_pmull_eor3_3way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL+SHA3 before selecting this kernel. +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] +pub(super) fn crc16_ibm_pmull_eor3_3way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL and SHA3 support before invoking this private wrapper. unsafe { crc16_width32_pmull_eor3_3way( crc, @@ -1194,8 +1342,11 @@ mod tests { const STATES: &[u16] = &[0, 0x1d0f, 0xa5a5, u16::MAX]; fn data() -> Vec { - (0..4111) - .map(|i| (i as u8).wrapping_mul(29).wrapping_add((i >> 8) as u8)) + (0u16..4111) + .map(|i| { + let [low, high] = i.to_le_bytes(); + low.wrapping_mul(29).wrapping_add(high) + }) .collect() } @@ -1204,7 +1355,7 @@ mod tests { for &state in STATES { for &offset in OFFSETS { for &len in LENS { - let slice = &input[offset..offset + len]; + let slice = &input[offset..offset.strict_add(len)]; assert_eq!( kernel(state, slice), portable(state, slice), diff --git a/src/checksum/crc16/config.rs b/src/checksum/crc16/config.rs index 2649ddb1..da879f6a 100644 --- a/src/checksum/crc16/config.rs +++ b/src/checksum/crc16/config.rs @@ -95,8 +95,16 @@ fn parse_force_value(value: &str) -> Crc16Force { #[inline] #[must_use] -#[allow(unused_variables)] fn clamp_force_to_caps(requested: Crc16Force, caps: Caps) -> Crc16Force { + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" + )))] + let _ = caps; + match requested { Crc16Force::Auto | Crc16Force::Reference | Crc16Force::Portable => requested, Crc16Force::Clmul => { @@ -188,7 +196,7 @@ fn config_ibm(caps: Caps) -> Crc16Config { /// detected platform capabilities. #[inline] #[must_use] -pub fn get_ccitt() -> Crc16Config { +pub(super) fn get_ccitt() -> Crc16Config { #[cfg(feature = "std")] { use std::sync::OnceLock; @@ -208,7 +216,7 @@ pub fn get_ccitt() -> Crc16Config { /// detected platform capabilities. #[inline] #[must_use] -pub fn get_ibm() -> Crc16Config { +pub(super) fn get_ibm() -> Crc16Config { #[cfg(feature = "std")] { use std::sync::OnceLock; diff --git a/src/checksum/crc16/kernels.rs b/src/checksum/crc16/kernels.rs index 2ea447b6..09589ac2 100644 --- a/src/checksum/crc16/kernels.rs +++ b/src/checksum/crc16/kernels.rs @@ -14,24 +14,24 @@ //! - Tier 4 (Wide): VPCLMUL (x86_64 AVX-512), Zvbc (riscv64) /// Reference (bitwise) kernel name. -pub use kernels::REFERENCE; +pub(in crate::checksum) use kernels::REFERENCE; use crate::checksum::common::kernels; /// Portable slice-by-8 kernel name. -pub const PORTABLE_SLICE8: &str = kernels::PORTABLE_SLICE8; +pub(in crate::checksum) const PORTABLE_SLICE8: &str = kernels::PORTABLE_SLICE8; // Kernel Name Tables and Functions (per architecture) #[cfg(target_arch = "x86_64")] -pub mod x86_64 { +pub(in crate::checksum) mod x86_64 { use super::super::x86_64 as arch; use crate::checksum::dispatchers::Crc16Fn; // CRC-16/CCITT Kernel Functions /// CCITT PCLMUL kernel. - pub const CCITT_PCLMUL: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_PCLMUL: [Crc16Fn; 5] = [ arch::crc16_ccitt_pclmul_safe, arch::crc16_ccitt_pclmul_2way_safe, arch::crc16_ccitt_pclmul_4way_safe, @@ -40,10 +40,10 @@ pub mod x86_64 { ]; /// CCITT PCLMUL small-buffer kernel. - pub const CCITT_PCLMUL_SMALL_KERNEL: Crc16Fn = arch::crc16_ccitt_pclmul_small_safe; + pub(in crate::checksum) const CCITT_PCLMUL_SMALL_KERNEL: Crc16Fn = arch::crc16_ccitt_pclmul_small_safe; /// CCITT VPCLMUL kernel. - pub const CCITT_VPCLMUL: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_VPCLMUL: [Crc16Fn; 5] = [ arch::crc16_ccitt_vpclmul_safe, arch::crc16_ccitt_vpclmul_2way_safe, arch::crc16_ccitt_vpclmul_4way_safe, @@ -54,7 +54,7 @@ pub mod x86_64 { // CRC-16/IBM Kernel Functions /// IBM PCLMUL kernel. - pub const IBM_PCLMUL: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_PCLMUL: [Crc16Fn; 5] = [ arch::crc16_ibm_pclmul_safe, arch::crc16_ibm_pclmul_2way_safe, arch::crc16_ibm_pclmul_4way_safe, @@ -63,10 +63,10 @@ pub mod x86_64 { ]; /// IBM PCLMUL small-buffer kernel. - pub const IBM_PCLMUL_SMALL_KERNEL: Crc16Fn = arch::crc16_ibm_pclmul_small_safe; + pub(in crate::checksum) const IBM_PCLMUL_SMALL_KERNEL: Crc16Fn = arch::crc16_ibm_pclmul_small_safe; /// IBM VPCLMUL kernel. - pub const IBM_VPCLMUL: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_VPCLMUL: [Crc16Fn; 5] = [ arch::crc16_ibm_vpclmul_safe, arch::crc16_ibm_vpclmul_2way_safe, arch::crc16_ibm_vpclmul_4way_safe, @@ -76,14 +76,14 @@ pub mod x86_64 { } #[cfg(target_arch = "aarch64")] -pub mod aarch64 { +pub(in crate::checksum) mod aarch64 { use super::super::aarch64 as arch; use crate::checksum::dispatchers::Crc16Fn; // CRC-16/CCITT Kernel Functions /// CCITT PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)]. - pub const CCITT_PMULL: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_PMULL: [Crc16Fn; 5] = [ arch::crc16_ccitt_pmull_safe, arch::crc16_ccitt_pmull_2way_safe, arch::crc16_ccitt_pmull_3way_safe, @@ -92,8 +92,8 @@ pub mod aarch64 { ]; /// CCITT PMULL+EOR3 kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)]. - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] - pub const CCITT_PMULL_EOR3: [Crc16Fn; 5] = [ + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] + pub(in crate::checksum) const CCITT_PMULL_EOR3: [Crc16Fn; 5] = [ arch::crc16_ccitt_pmull_eor3_safe, arch::crc16_ccitt_pmull_eor3_2way_safe, arch::crc16_ccitt_pmull_eor3_3way_safe, @@ -102,12 +102,12 @@ pub mod aarch64 { ]; /// CCITT PMULL small-buffer kernel. - pub const CCITT_PMULL_SMALL_KERNEL: Crc16Fn = arch::crc16_ccitt_pmull_small_safe; + pub(in crate::checksum) const CCITT_PMULL_SMALL_KERNEL: Crc16Fn = arch::crc16_ccitt_pmull_small_safe; // CRC-16/IBM Kernel Functions /// IBM PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)]. - pub const IBM_PMULL: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_PMULL: [Crc16Fn; 5] = [ arch::crc16_ibm_pmull_safe, arch::crc16_ibm_pmull_2way_safe, arch::crc16_ibm_pmull_3way_safe, @@ -116,8 +116,8 @@ pub mod aarch64 { ]; /// IBM PMULL+EOR3 kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)]. - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] - pub const IBM_PMULL_EOR3: [Crc16Fn; 5] = [ + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] + pub(in crate::checksum) const IBM_PMULL_EOR3: [Crc16Fn; 5] = [ arch::crc16_ibm_pmull_eor3_safe, arch::crc16_ibm_pmull_eor3_2way_safe, arch::crc16_ibm_pmull_eor3_3way_safe, @@ -126,16 +126,16 @@ pub mod aarch64 { ]; /// IBM PMULL small-buffer kernel. - pub const IBM_PMULL_SMALL_KERNEL: Crc16Fn = arch::crc16_ibm_pmull_small_safe; + pub(in crate::checksum) const IBM_PMULL_SMALL_KERNEL: Crc16Fn = arch::crc16_ibm_pmull_small_safe; } #[cfg(target_arch = "powerpc64")] -pub mod power { +pub(in crate::checksum) mod power { use super::super::power as arch; use crate::checksum::dispatchers::Crc16Fn; /// CCITT VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)]. - pub const CCITT_VPMSUM: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_VPMSUM: [Crc16Fn; 5] = [ arch::crc16_ccitt_vpmsum_safe, arch::crc16_ccitt_vpmsum_2way_safe, arch::crc16_ccitt_vpmsum_4way_safe, @@ -144,7 +144,7 @@ pub mod power { ]; /// IBM VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)]. - pub const IBM_VPMSUM: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_VPMSUM: [Crc16Fn; 5] = [ arch::crc16_ibm_vpmsum_safe, arch::crc16_ibm_vpmsum_2way_safe, arch::crc16_ibm_vpmsum_4way_safe, @@ -154,12 +154,12 @@ pub mod power { } #[cfg(target_arch = "s390x")] -pub mod s390x { +pub(in crate::checksum) mod s390x { use super::super::s390x as arch; use crate::checksum::dispatchers::Crc16Fn; /// CCITT VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CCITT_VGFM: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_VGFM: [Crc16Fn; 5] = [ arch::crc16_ccitt_vgfm_safe, arch::crc16_ccitt_vgfm_2way_safe, arch::crc16_ccitt_vgfm_4way_safe, @@ -168,7 +168,7 @@ pub mod s390x { ]; /// IBM VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const IBM_VGFM: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_VGFM: [Crc16Fn; 5] = [ arch::crc16_ibm_vgfm_safe, arch::crc16_ibm_vgfm_2way_safe, arch::crc16_ibm_vgfm_4way_safe, @@ -178,31 +178,12 @@ pub mod s390x { } #[cfg(target_arch = "riscv64")] -#[allow(dead_code)] -pub mod riscv64 { +pub(in crate::checksum) mod riscv64 { use super::super::riscv64 as arch; use crate::checksum::dispatchers::Crc16Fn; - /// Zbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const ZBC_NAMES: &[&str] = &[ - "riscv64/zbc", - "riscv64/zbc-2way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", - ]; - - /// Zvbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const ZVBC_NAMES: &[&str] = &[ - "riscv64/zvbc", - "riscv64/zvbc-2way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - ]; - /// CCITT Zbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CCITT_ZBC: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_ZBC: [Crc16Fn; 5] = [ arch::crc16_ccitt_zbc_safe, arch::crc16_ccitt_zbc_2way_safe, arch::crc16_ccitt_zbc_4way_safe, @@ -211,7 +192,7 @@ pub mod riscv64 { ]; /// CCITT Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CCITT_ZVBC: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_ZVBC: [Crc16Fn; 5] = [ arch::crc16_ccitt_zvbc_safe, arch::crc16_ccitt_zvbc_2way_safe, arch::crc16_ccitt_zvbc_4way_safe, @@ -220,7 +201,7 @@ pub mod riscv64 { ]; /// IBM Zbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const IBM_ZBC: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_ZBC: [Crc16Fn; 5] = [ arch::crc16_ibm_zbc_safe, arch::crc16_ibm_zbc_2way_safe, arch::crc16_ibm_zbc_4way_safe, @@ -229,7 +210,7 @@ pub mod riscv64 { ]; /// IBM Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const IBM_ZVBC: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_ZVBC: [Crc16Fn; 5] = [ arch::crc16_ibm_zvbc_safe, arch::crc16_ibm_zvbc_2way_safe, arch::crc16_ibm_zvbc_4way_safe, diff --git a/src/checksum/crc16/keys.rs b/src/checksum/crc16/keys.rs index 29b3ea09..cf4e8809 100644 --- a/src/checksum/crc16/keys.rs +++ b/src/checksum/crc16/keys.rs @@ -10,7 +10,7 @@ use crate::checksum::common::tables::{CRC16_CCITT_POLY, CRC16_IBM_POLY}; /// Key schedule for CRC-16/CCITT (X.25 / IBM-SDLC), reflected polynomial. #[rustfmt::skip] -pub(crate) const CRC16_CCITT_KEYS_REFLECTED: [u64; 23] = [ +pub(super) const CRC16_CCITT_KEYS_REFLECTED: [u64; 23] = [ 0x0000000000000000, 0x00000000000189ae, 0x0000000000008e10, @@ -38,7 +38,7 @@ pub(crate) const CRC16_CCITT_KEYS_REFLECTED: [u64; 23] = [ /// Key schedule for CRC-16/IBM (ARC), reflected polynomial. #[rustfmt::skip] -pub(crate) const CRC16_IBM_KEYS_REFLECTED: [u64; 23] = [ +pub(super) const CRC16_IBM_KEYS_REFLECTED: [u64; 23] = [ 0x0000000000000000, 0x0000000000018cc2, 0x000000000001d0c2, @@ -71,41 +71,73 @@ pub(crate) const CRC16_IBM_KEYS_REFLECTED: [u64; 23] = [ /// - advance a stream by `N * 128B` per iteration (striping) /// - merge streams back together (combine coefficients) #[derive(Clone, Copy, Debug)] -#[allow(dead_code)] // Field subsets vary by architecture (x86_64/aarch64/power/s390x/riscv64 stream widths). -pub(crate) struct Width32StreamConstants { +pub(super) struct Width32StreamConstants { /// 2-way fold coefficient (256B = 2×128B). - pub fold_256b: (u64, u64), + pub(super) fold_256b: (u64, u64), /// 3-way fold coefficient (384B = 3×128B). - pub fold_384b: (u64, u64), + #[cfg(target_arch = "aarch64")] + pub(super) fold_384b: (u64, u64), /// 4-way fold coefficient (512B = 4×128B). - pub fold_512b: (u64, u64), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] + pub(super) fold_512b: (u64, u64), /// 7-way fold coefficient (896B = 7×128B). - pub fold_896b: (u64, u64), + #[cfg(target_arch = "x86_64")] + pub(super) fold_896b: (u64, u64), /// 8-way fold coefficient (1024B = 8×128B). - pub fold_1024b: (u64, u64), + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] + pub(super) fold_1024b: (u64, u64), /// 4-way combine coefficients: shifts by 384B, 256B, 128B. - pub combine_4way: [(u64, u64); 3], + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] + pub(super) combine_4way: [(u64, u64); 3], /// 7-way combine coefficients: shifts by 768B, 640B, 512B, 384B, 256B, 128B. - pub combine_7way: [(u64, u64); 6], + #[cfg(target_arch = "x86_64")] + pub(super) combine_7way: [(u64, u64); 6], /// 8-way combine coefficients: shifts by 896B, 768B, 640B, 512B, 384B, 256B, 128B. - pub combine_8way: [(u64, u64); 7], + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] + pub(super) combine_8way: [(u64, u64); 7], } impl Width32StreamConstants { /// Compute all multi-stream folding constants for a given reflected polynomial. #[must_use] - pub const fn new(reflected_poly: u32) -> Self { + const fn new(reflected_poly: u32) -> Self { Self { fold_256b: fold16_coeff_for_bytes(reflected_poly, 256), + #[cfg(target_arch = "aarch64")] fold_384b: fold16_coeff_for_bytes(reflected_poly, 384), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] fold_512b: fold16_coeff_for_bytes(reflected_poly, 512), + #[cfg(target_arch = "x86_64")] fold_896b: fold16_coeff_for_bytes(reflected_poly, 896), + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] fold_1024b: fold16_coeff_for_bytes(reflected_poly, 1024), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] combine_4way: [ fold16_coeff_for_bytes(reflected_poly, 384), fold16_coeff_for_bytes(reflected_poly, 256), fold16_coeff_for_bytes(reflected_poly, 128), ], + #[cfg(target_arch = "x86_64")] combine_7way: [ fold16_coeff_for_bytes(reflected_poly, 768), fold16_coeff_for_bytes(reflected_poly, 640), @@ -114,6 +146,7 @@ impl Width32StreamConstants { fold16_coeff_for_bytes(reflected_poly, 256), fold16_coeff_for_bytes(reflected_poly, 128), ], + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] combine_8way: [ fold16_coeff_for_bytes(reflected_poly, 896), fold16_coeff_for_bytes(reflected_poly, 768), @@ -127,9 +160,9 @@ impl Width32StreamConstants { } } -pub(crate) const CRC16_CCITT_STREAM_REFLECTED: Width32StreamConstants = +pub(super) const CRC16_CCITT_STREAM_REFLECTED: Width32StreamConstants = Width32StreamConstants::new(CRC16_CCITT_POLY as u32); -pub(crate) const CRC16_IBM_STREAM_REFLECTED: Width32StreamConstants = +pub(super) const CRC16_IBM_STREAM_REFLECTED: Width32StreamConstants = Width32StreamConstants::new(CRC16_IBM_POLY as u32); // Constant Generation (compile-time) @@ -162,16 +195,16 @@ const fn reduce128(hi: u64, lo: u64, poly: u32) -> u32 { let poly_full: u128 = (1u128.strict_shl(32)) | (poly as u128); let mut val: u128 = (hi as u128).strict_shl(64) | (lo as u128); - let mut bit: i32 = 127; + let mut bit: u32 = 127; while bit >= 32 { - let b = bit as u32; - if ((val.strict_shr(b)) & 1) != 0 { - val ^= poly_full.strict_shl(b.strict_sub(32)); + if ((val.strict_shr(bit)) & 1) != 0 { + val ^= poly_full.strict_shl(bit.strict_sub(32)); } bit = bit.strict_sub(1); } - val as u32 + let [b0, b1, b2, b3, ..] = val.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) } /// Compute x^n mod (x^width + poly) in GF(2) where `poly` is the normal CRC polynomial diff --git a/src/checksum/crc16/mod.rs b/src/checksum/crc16/mod.rs index cdf4af50..210d31a2 100644 --- a/src/checksum/crc16/mod.rs +++ b/src/checksum/crc16/mod.rs @@ -4,26 +4,18 @@ //! - [`Crc16Ccitt`] - CRC-16/X25 (also known as IBM-SDLC) //! - [`Crc16Ibm`] - CRC-16/ARC (also known as CRC-16/IBM) //! -//! # Quick Start -//! -//! ```rust -//! use rscrypto::checksum::{Checksum, ChecksumCombine, Crc16Ccitt, Crc16Ibm}; -//! -//! let data = b"123456789"; -//! assert_eq!(Crc16Ccitt::checksum(data), 0x906E); -//! assert_eq!(Crc16Ibm::checksum(data), 0xBB3D); -//! -//! let (a, b) = data.split_at(4); -//! let combined = Crc16Ccitt::combine(Crc16Ccitt::checksum(a), Crc16Ccitt::checksum(b), b.len()); -//! assert_eq!(combined, Crc16Ccitt::checksum(data)); -//! ``` - pub(crate) mod config; pub(crate) mod kernels; +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" +))] pub(crate) mod keys; pub(crate) mod portable; -#[allow(unused_imports)] pub use config::{Crc16Config, Crc16Force}; #[cfg(any(test, feature = "std"))] @@ -32,10 +24,6 @@ use crate::checksum::common::{ combine::{Gf2Matrix16, combine_crc16, generate_shift8_matrix_16}, tables::{CRC16_CCITT_POLY, CRC16_IBM_POLY, generate_crc16_tables_8}, }; -// Re-export traits for test modules (`use super::*`). -#[allow(unused_imports)] -pub(super) use crate::traits::{Checksum, ChecksumCombine}; - #[cfg(target_arch = "aarch64")] mod aarch64; #[cfg(target_arch = "powerpc64")] @@ -52,9 +40,9 @@ mod x86_64; mod kernel_tables { use super::*; - pub static CCITT_TABLES_8: [[u16; 256]; 8] = generate_crc16_tables_8(CRC16_CCITT_POLY); + pub(super) static CCITT_TABLES_8: [[u16; 256]; 8] = generate_crc16_tables_8(CRC16_CCITT_POLY); - pub static IBM_TABLES_8: [[u16; 256]; 8] = generate_crc16_tables_8(CRC16_IBM_POLY); + pub(super) static IBM_TABLES_8: [[u16; 256]; 8] = generate_crc16_tables_8(CRC16_IBM_POLY); } // Reference Kernel Wrappers @@ -404,6 +392,7 @@ impl crate::traits::ChecksumCombine for Crc16Ccitt { #[cfg(feature = "alloc")] impl Crc16Ccitt { + /// Creates a buffering wrapper that coalesces short updates before CRC-16/X25 dispatch. #[must_use] pub fn buffered() -> BufferedCrc16Ccitt { BufferedCrc16Ccitt::new() @@ -546,6 +535,7 @@ impl crate::traits::ChecksumCombine for Crc16Ibm { #[cfg(feature = "alloc")] impl Crc16Ibm { + /// Creates a buffering wrapper that coalesces short updates before CRC-16/ARC dispatch. #[must_use] pub fn buffered() -> BufferedCrc16Ibm { BufferedCrc16Ibm::new() @@ -611,6 +601,7 @@ mod tests { extern crate std; use super::*; + use crate::traits::{Checksum, ChecksumCombine}; #[test] fn test_vectors_crc16_ccitt_x25() { @@ -694,6 +685,7 @@ mod cross_check { use super::*; use crate::checksum::common::tests::{STREAMING_CHUNK_SIZES, TEST_LENGTHS, generate_test_data}; + use crate::traits::{Checksum, ChecksumCombine}; // CRC-16/CCITT Cross-Check Tests diff --git a/src/checksum/crc16/portable.rs b/src/checksum/crc16/portable.rs index 650b3db5..1da6d283 100644 --- a/src/checksum/crc16/portable.rs +++ b/src/checksum/crc16/portable.rs @@ -5,13 +5,13 @@ use crate::checksum::common::portable; /// CRC-16/CCITT (X25 / IBM-SDLC) slice-by-8 computation. #[inline] -pub fn crc16_ccitt_slice8(crc: u16, data: &[u8]) -> u16 { +pub(in crate::checksum) fn crc16_ccitt_slice8(crc: u16, data: &[u8]) -> u16 { portable::slice8_16(crc, data, &kernel_tables::CCITT_TABLES_8) } /// CRC-16/IBM (ARC) slice-by-8 computation. #[inline] -pub fn crc16_ibm_slice8(crc: u16, data: &[u8]) -> u16 { +pub(in crate::checksum) fn crc16_ibm_slice8(crc: u16, data: &[u8]) -> u16 { portable::slice8_16(crc, data, &kernel_tables::IBM_TABLES_8) } @@ -21,7 +21,7 @@ pub fn crc16_ibm_slice8(crc: u16, data: &[u8]) -> u16 { /// /// Uses one 256-entry table rather than the slice-by-8 table set. #[inline(always)] -pub fn crc16_ccitt_bytewise(crc: u16, data: &[u8]) -> u16 { +pub(in crate::checksum) fn crc16_ccitt_bytewise(crc: u16, data: &[u8]) -> u16 { crc16_bytewise(crc, data, &kernel_tables::CCITT_TABLES_8[0]) } @@ -29,13 +29,12 @@ pub fn crc16_ccitt_bytewise(crc: u16, data: &[u8]) -> u16 { /// /// Uses one 256-entry table rather than the slice-by-8 table set. #[inline(always)] -pub fn crc16_ibm_bytewise(crc: u16, data: &[u8]) -> u16 { +pub(in crate::checksum) fn crc16_ibm_bytewise(crc: u16, data: &[u8]) -> u16 { crc16_bytewise(crc, data, &kernel_tables::IBM_TABLES_8[0]) } /// Update CRC-16 state using a byte-at-a-time lookup table. #[inline(always)] -#[allow(clippy::indexing_slicing)] // index is 0..=255 by mask, table is [u16; 256] fn crc16_bytewise(mut crc: u16, data: &[u8], table: &[u16; 256]) -> u16 { for &b in data { let index = ((crc ^ (b as u16)) & 0xFF) as usize; diff --git a/src/checksum/crc16/power.rs b/src/checksum/crc16/power.rs index 5e145fa5..aa8f0720 100644 --- a/src/checksum/crc16/power.rs +++ b/src/checksum/crc16/power.rs @@ -8,10 +8,7 @@ //! Uses `unsafe` for Power SIMD + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher // SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, @@ -19,6 +16,8 @@ use core::{ simd::i64x2, }; +use crate::checksum::common::{low_u16, low_u32}; + use super::keys::{ CRC16_CCITT_KEYS_REFLECTED, CRC16_CCITT_STREAM_REFLECTED, CRC16_IBM_KEYS_REFLECTED, CRC16_IBM_STREAM_REFLECTED, }; @@ -80,12 +79,13 @@ impl Simd { /// On `powerpc64le` this is a no-op. On big-endian `powerpc64`, we byte-swap /// each 64-bit lane so the folding algorithm sees the same lane values as on /// little-endian platforms. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn to_le(self) -> Self { - // SAFETY: Caller guarantees: - // 1. ALTIVEC + VSX + POWER8-VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. #[cfg(target_endian = "little")] { self @@ -107,6 +107,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -132,6 +135,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -153,6 +159,10 @@ impl Simd { /// Fold 16 bytes (reflected width32 folding primitive): /// `self.low ⊗ coeff.high ⊕ self.high ⊗ coeff.low`. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -170,6 +180,9 @@ impl Simd { unsafe { Self(Self::vpmsumd(self.0, coeff.swap_lanes().0)) } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -186,6 +199,10 @@ impl Simd { } /// Fold 16 bytes down to the "width32" reduction state (reflected mode). + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -216,6 +233,10 @@ impl Simd { } /// Barrett reduction for reflected width32; returns the updated CRC state. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -232,11 +253,14 @@ impl Simd { unsafe { let t1 = Self::mul64(self.low_64(), mu); let l = Self::mul64(t1.low_64(), poly); - (self ^ l).high_64() as u32 + low_u32((self ^ l).high_64()) } } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -264,6 +288,9 @@ unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u3 } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -305,6 +332,9 @@ unsafe fn update_simd_width32_reflected(state: u32, first: &[Simd; 8], rest: &[[ } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -329,6 +359,9 @@ unsafe fn fold_block_128_reflected(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -354,6 +387,9 @@ unsafe fn normalize_block_le(mut block: [Simd; 8]) -> [Simd; 8] { } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -416,6 +452,9 @@ unsafe fn update_simd_width32_reflected_2way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -444,7 +483,7 @@ unsafe fn update_simd_width32_reflected_4way( return update_simd_width32_reflected(state, first, rest, keys); } - let aligned = (blocks.len() / 4) * 4; + let aligned = blocks.len() & !3_usize; let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); let coeff_128 = Simd::new(keys[4], keys[3]); @@ -505,6 +544,9 @@ unsafe fn update_simd_width32_reflected_4way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -533,7 +575,7 @@ unsafe fn update_simd_width32_reflected_8way( return update_simd_width32_reflected(state, first, rest, keys); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len() & !7_usize; let coeff_1024 = Simd::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd::new(keys[4], keys[3]); @@ -642,6 +684,9 @@ unsafe fn update_simd_width32_reflected_8way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -662,11 +707,14 @@ unsafe fn crc16_width32_vpmsum(mut state: u16, data: &[u8], keys: &[u64; 23], po state = portable(state, left); let state32 = update_simd_width32_reflected(state as u32, first, rest, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -693,11 +741,14 @@ unsafe fn crc16_width32_vpmsum_2way( state = portable(state, left); let state32 = update_simd_width32_reflected_2way(state as u32, middle, fold_256b, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -725,11 +776,14 @@ unsafe fn crc16_width32_vpmsum_4way( state = portable(state, left); let state32 = update_simd_width32_reflected_4way(state as u32, middle, fold_512b, combine, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -757,7 +811,7 @@ unsafe fn crc16_width32_vpmsum_8way( state = portable(state, left); let state32 = update_simd_width32_reflected_8way(state as u32, middle, fold_1024b, combine, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } @@ -766,11 +820,9 @@ unsafe fn crc16_width32_vpmsum_8way( /// CRC-16/CCITT VPMSUMD kernel. /// -/// # Safety -/// -/// Dispatcher verifies VPMSUMD before selecting this kernel. +/// Runtime dispatch selects this kernel only when VPMSUMD is available. #[inline] -pub fn crc16_ccitt_vpmsum_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpmsum_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum( @@ -783,7 +835,7 @@ pub fn crc16_ccitt_vpmsum_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_vpmsum_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpmsum_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum_2way( @@ -797,7 +849,7 @@ pub fn crc16_ccitt_vpmsum_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_vpmsum_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpmsum_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum_4way( @@ -812,7 +864,7 @@ pub fn crc16_ccitt_vpmsum_4way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_vpmsum_8way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpmsum_8way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum_8way( @@ -828,17 +880,15 @@ pub fn crc16_ccitt_vpmsum_8way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM VPMSUMD kernel. /// -/// # Safety -/// -/// Dispatcher verifies VPMSUMD before selecting this kernel. +/// Runtime dispatch selects this kernel only when VPMSUMD is available. #[inline] -pub fn crc16_ibm_vpmsum_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpmsum_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } #[inline] -pub fn crc16_ibm_vpmsum_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpmsum_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum_2way( @@ -852,7 +902,7 @@ pub fn crc16_ibm_vpmsum_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_vpmsum_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpmsum_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum_4way( @@ -867,7 +917,7 @@ pub fn crc16_ibm_vpmsum_4way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_vpmsum_8way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpmsum_8way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum_8way( diff --git a/src/checksum/crc16/riscv64.rs b/src/checksum/crc16/riscv64.rs index f042f18b..4c8430ab 100644 --- a/src/checksum/crc16/riscv64.rs +++ b/src/checksum/crc16/riscv64.rs @@ -8,17 +8,14 @@ //! Uses `unsafe` for RISC-V inline assembly. Callers must ensure the required //! CPU features are available before executing the accelerated path (the //! dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, - mem::MaybeUninit, ops::{BitXor, BitXorAssign}, }; +use crate::checksum::common::{low_u16, low_u32}; + use super::keys::{ CRC16_CCITT_KEYS_REFLECTED, CRC16_CCITT_STREAM_REFLECTED, CRC16_IBM_KEYS_REFLECTED, CRC16_IBM_STREAM_REFLECTED, }; @@ -70,11 +67,11 @@ impl Simd { // Zbc carryless multiply primitives #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn clmul_lo(a: u64, b: u64) -> u64 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + fn clmul_lo(a: u64, b: u64) -> u64 { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -91,11 +88,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn clmul_hi(a: u64, b: u64) -> u64 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + fn clmul_hi(a: u64, b: u64) -> u64 { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -112,18 +109,14 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - Self { - hi: Self::clmul_hi(a, b), - lo: Self::clmul_lo(a, b), - } + fn mul64(a: u64, b: u64) -> Self { + Self { + hi: Self::clmul_hi(a, b), + lo: Self::clmul_lo(a, b), } } } @@ -132,34 +125,20 @@ impl Simd { #[inline(always)] fn load_block(block: &Block) -> [Simd; 8] { - let mut out = MaybeUninit::<[Simd; 8]>::uninit(); - let base = out.as_mut_ptr().cast::(); - - let mut i = 0usize; - while i < 8 { - let lo = u64::from_le(block[i * 2]); - let hi = u64::from_le(block[i * 2 + 1]); - // SAFETY: `base` points to a `[Simd; 8]` buffer and `i` is in-bounds. - unsafe { - base.add(i).write(Simd::new(hi, lo)); - } - i = i.strict_add(1); + let mut out = [Simd::new(0, 0); 8]; + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new(u64::from_le(high), u64::from_le(low)); } - - // SAFETY: all 8 elements are initialized above. - unsafe { out.assume_init() } + out } #[inline] fn load_block_split(block: &Block) -> ([u64; 8], [u64; 8]) { let mut hi = [0u64; 8]; let mut lo = [0u64; 8]; - - let mut i = 0usize; - while i < 8 { - lo[i] = u64::from_le(block[i * 2]); - hi[i] = u64::from_le(block[i * 2 + 1]); - i = i.strict_add(1); + for ((hi_lane, lo_lane), &[low, high]) in hi.iter_mut().zip(&mut lo).zip(block.as_chunks::<2>().0) { + *hi_lane = u64::from_le(high); + *lo_lane = u64::from_le(low); } (hi, lo) @@ -168,277 +147,249 @@ fn load_block_split(block: &Block) -> ([u64; 8], [u64; 8]) { // ZBC (scalar carryless multiply) backend #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_16_zbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - Simd::mul64(x.low_64(), coeff_high) ^ Simd::mul64(x.high_64(), coeff_low) - } +fn fold_16_zbc(x: Simd, coeff: (u64, u64)) -> Simd { + let (coeff_high, coeff_low) = coeff; + Simd::mul64(x.low_64(), coeff_high) ^ Simd::mul64(x.high_64(), coeff_low) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_16_reflected_zbc(x: Simd, coeff: (u64, u64), data_to_xor: Simd) -> Simd { - // SAFETY: Caller guarantees: - // 1. ZBC + ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { data_to_xor ^ fold_16_zbc(x, coeff) } +fn fold_16_reflected_zbc(x: Simd, coeff: (u64, u64), data_to_xor: Simd) -> Simd { + data_to_xor ^ fold_16_zbc(x, coeff) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_width32_reflected_zbc(x: Simd, high: u64, low: u64) -> Simd { - // SAFETY: Caller guarantees: - // 1. ZBC + ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let clmul = Simd::mul64(x.low_64(), low); - let shifted = Simd::new(0, x.high_64()); - let mut state = clmul ^ shifted; +fn fold_width32_reflected_zbc(x: Simd, high: u64, low: u64) -> Simd { + let clmul = Simd::mul64(x.low_64(), low); + let shifted = Simd::new(0, x.high_64()); + let mut state = clmul ^ shifted; - let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = Simd::mul64(shifted_high, high); - state = clmul ^ masked; + let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = Simd::mul64(shifted_high, high); + state = clmul ^ masked; - state - } + state } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn barrett_width32_reflected_zbc(x: Simd, poly: u64, mu: u64) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = Simd::mul64(x.low_64(), mu); - let l = Simd::mul64(t1.low_64(), poly); - (x ^ l).high_64() as u32 - } +fn barrett_width32_reflected_zbc(x: Simd, poly: u64, mu: u64) -> u32 { + let t1 = Simd::mul64(x.low_64(), mu); + let l = Simd::mul64(t1.low_64(), poly); + low_u32((x ^ l).high_64()) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn finalize_lanes_width32_reflected_zbc(x: [Simd; 8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut res = x[7]; - res = fold_16_reflected_zbc(x[0], (keys[10], keys[9]), res); - res = fold_16_reflected_zbc(x[1], (keys[12], keys[11]), res); - res = fold_16_reflected_zbc(x[2], (keys[14], keys[13]), res); - res = fold_16_reflected_zbc(x[3], (keys[16], keys[15]), res); - res = fold_16_reflected_zbc(x[4], (keys[18], keys[17]), res); - res = fold_16_reflected_zbc(x[5], (keys[20], keys[19]), res); - res = fold_16_reflected_zbc(x[6], (keys[2], keys[1]), res); - - barrett_width32_reflected_zbc(fold_width32_reflected_zbc(res, keys[6], keys[5]), keys[8], keys[7]) - } +fn finalize_lanes_width32_reflected_zbc(x: [Simd; 8], keys: &[u64; 23]) -> u32 { + let mut res = x[7]; + res = fold_16_reflected_zbc(x[0], (keys[10], keys[9]), res); + res = fold_16_reflected_zbc(x[1], (keys[12], keys[11]), res); + res = fold_16_reflected_zbc(x[2], (keys[14], keys[13]), res); + res = fold_16_reflected_zbc(x[3], (keys[16], keys[15]), res); + res = fold_16_reflected_zbc(x[4], (keys[18], keys[17]), res); + res = fold_16_reflected_zbc(x[5], (keys[20], keys[19]), res); + res = fold_16_reflected_zbc(x[6], (keys[2], keys[1]), res); + + barrett_width32_reflected_zbc(fold_width32_reflected_zbc(res, keys[6], keys[5]), keys[8], keys[7]) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut x = load_block(first); - x[0] ^= Simd::new(0, state as u64); - - let coeff_128b = (keys[4], keys[3]); - for block in rest { - let chunk = load_block(block); - x[0] = fold_16_reflected_zbc(x[0], coeff_128b, chunk[0]); - x[1] = fold_16_reflected_zbc(x[1], coeff_128b, chunk[1]); - x[2] = fold_16_reflected_zbc(x[2], coeff_128b, chunk[2]); - x[3] = fold_16_reflected_zbc(x[3], coeff_128b, chunk[3]); - x[4] = fold_16_reflected_zbc(x[4], coeff_128b, chunk[4]); - x[5] = fold_16_reflected_zbc(x[5], coeff_128b, chunk[5]); - x[6] = fold_16_reflected_zbc(x[6], coeff_128b, chunk[6]); - x[7] = fold_16_reflected_zbc(x[7], coeff_128b, chunk[7]); - } +fn update_simd_zbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { + let mut x = load_block(first); + x[0] ^= Simd::new(0, state as u64); - finalize_lanes_width32_reflected_zbc(x, keys) + let coeff_128b = (keys[4], keys[3]); + for block in rest { + let chunk = load_block(block); + x[0] = fold_16_reflected_zbc(x[0], coeff_128b, chunk[0]); + x[1] = fold_16_reflected_zbc(x[1], coeff_128b, chunk[1]); + x[2] = fold_16_reflected_zbc(x[2], coeff_128b, chunk[2]); + x[3] = fold_16_reflected_zbc(x[3], coeff_128b, chunk[3]); + x[4] = fold_16_reflected_zbc(x[4], coeff_128b, chunk[4]); + x[5] = fold_16_reflected_zbc(x[5], coeff_128b, chunk[5]); + x[6] = fold_16_reflected_zbc(x[6], coeff_128b, chunk[6]); + x[7] = fold_16_reflected_zbc(x[7], coeff_128b, chunk[7]); } + + finalize_lanes_width32_reflected_zbc(x, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_block_128_reflected_zbc(x: &mut [Simd; 8], block: &Block, coeff: (u64, u64)) { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let chunk = load_block(block); - x[0] = fold_16_reflected_zbc(x[0], coeff, chunk[0]); - x[1] = fold_16_reflected_zbc(x[1], coeff, chunk[1]); - x[2] = fold_16_reflected_zbc(x[2], coeff, chunk[2]); - x[3] = fold_16_reflected_zbc(x[3], coeff, chunk[3]); - x[4] = fold_16_reflected_zbc(x[4], coeff, chunk[4]); - x[5] = fold_16_reflected_zbc(x[5], coeff, chunk[5]); - x[6] = fold_16_reflected_zbc(x[6], coeff, chunk[6]); - x[7] = fold_16_reflected_zbc(x[7], coeff, chunk[7]); - } +fn fold_block_128_reflected_zbc(x: &mut [Simd; 8], block: &Block, coeff: (u64, u64)) { + let chunk = load_block(block); + x[0] = fold_16_reflected_zbc(x[0], coeff, chunk[0]); + x[1] = fold_16_reflected_zbc(x[1], coeff, chunk[1]); + x[2] = fold_16_reflected_zbc(x[2], coeff, chunk[2]); + x[3] = fold_16_reflected_zbc(x[3], coeff, chunk[3]); + x[4] = fold_16_reflected_zbc(x[4], coeff, chunk[4]); + x[5] = fold_16_reflected_zbc(x[5], coeff, chunk[5]); + x[6] = fold_16_reflected_zbc(x[6], coeff, chunk[6]); + x[7] = fold_16_reflected_zbc(x[7], coeff, chunk[7]); } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); +fn update_simd_zbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { + debug_assert!(!blocks.is_empty()); - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zbc(state, first, rest, keys); - } - - let even = blocks.len() & !1usize; - let coeff_256 = fold_256b; - let coeff_128 = (keys[4], keys[3]); + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zbc(state, first, rest, keys); + } - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); + let even = blocks.len() & !1usize; + let coeff_256 = fold_256b; + let coeff_128 = (keys[4], keys[3]); - s0[0] ^= Simd::new(0, state as u64); + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); - let mut i: usize = 2; - while i < even { - fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_256); - fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_256); - i = i.strict_add(2); - } + s0[0] ^= Simd::new(0, state as u64); - // Merge: A·s0 ⊕ s1 (A = shift by 128B). - s1[0] = fold_16_reflected_zbc(s0[0], coeff_128, s1[0]); - s1[1] = fold_16_reflected_zbc(s0[1], coeff_128, s1[1]); - s1[2] = fold_16_reflected_zbc(s0[2], coeff_128, s1[2]); - s1[3] = fold_16_reflected_zbc(s0[3], coeff_128, s1[3]); - s1[4] = fold_16_reflected_zbc(s0[4], coeff_128, s1[4]); - s1[5] = fold_16_reflected_zbc(s0[5], coeff_128, s1[5]); - s1[6] = fold_16_reflected_zbc(s0[6], coeff_128, s1[6]); - s1[7] = fold_16_reflected_zbc(s0[7], coeff_128, s1[7]); - - if even != blocks.len() { - fold_block_128_reflected_zbc(&mut s1, &blocks[even], coeff_128); - } + let mut i: usize = 2; + while i < even { + fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_256); + fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_256); + i = i.strict_add(2); + } - finalize_lanes_width32_reflected_zbc(s1, keys) + // Merge: A·s0 ⊕ s1 (A = shift by 128B). + s1[0] = fold_16_reflected_zbc(s0[0], coeff_128, s1[0]); + s1[1] = fold_16_reflected_zbc(s0[1], coeff_128, s1[1]); + s1[2] = fold_16_reflected_zbc(s0[2], coeff_128, s1[2]); + s1[3] = fold_16_reflected_zbc(s0[3], coeff_128, s1[3]); + s1[4] = fold_16_reflected_zbc(s0[4], coeff_128, s1[4]); + s1[5] = fold_16_reflected_zbc(s0[5], coeff_128, s1[5]); + s1[6] = fold_16_reflected_zbc(s0[6], coeff_128, s1[6]); + s1[7] = fold_16_reflected_zbc(s0[7], coeff_128, s1[7]); + + if even != blocks.len() { + fold_block_128_reflected_zbc(&mut s1, &blocks[even], coeff_128); } + + finalize_lanes_width32_reflected_zbc(s1, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc_4way( +fn update_simd_zbc_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zbc(state, first, rest, keys); - } + debug_assert!(!blocks.is_empty()); - let aligned = blocks.len().strict_div(4).strict_mul(4); + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zbc(state, first, rest, keys); + } - let coeff_512 = fold_512b; - let coeff_128 = (keys[4], keys[3]); - let c384 = combine[0]; - let c256 = combine[1]; - let c128 = combine[2]; + let aligned = blocks.len().strict_div(4).strict_mul(4); - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - let mut s2 = load_block(&blocks[2]); - let mut s3 = load_block(&blocks[3]); + let coeff_512 = fold_512b; + let coeff_128 = (keys[4], keys[3]); + let c384 = combine[0]; + let c256 = combine[1]; + let c128 = combine[2]; - s0[0] ^= Simd::new(0, state as u64); + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + let mut s2 = load_block(&blocks[2]); + let mut s3 = load_block(&blocks[3]); - let mut i: usize = 4; - while i < aligned { - fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_512); - fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_512); - fold_block_128_reflected_zbc(&mut s2, &blocks[i.strict_add(2)], coeff_512); - fold_block_128_reflected_zbc(&mut s3, &blocks[i.strict_add(3)], coeff_512); - i = i.strict_add(4); - } + s0[0] ^= Simd::new(0, state as u64); - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - s3[0] = fold_16_reflected_zbc(s2[0], c128, s3[0]); - s3[1] = fold_16_reflected_zbc(s2[1], c128, s3[1]); - s3[2] = fold_16_reflected_zbc(s2[2], c128, s3[2]); - s3[3] = fold_16_reflected_zbc(s2[3], c128, s3[3]); - s3[4] = fold_16_reflected_zbc(s2[4], c128, s3[4]); - s3[5] = fold_16_reflected_zbc(s2[5], c128, s3[5]); - s3[6] = fold_16_reflected_zbc(s2[6], c128, s3[6]); - s3[7] = fold_16_reflected_zbc(s2[7], c128, s3[7]); - - s3[0] = fold_16_reflected_zbc(s1[0], c256, s3[0]); - s3[1] = fold_16_reflected_zbc(s1[1], c256, s3[1]); - s3[2] = fold_16_reflected_zbc(s1[2], c256, s3[2]); - s3[3] = fold_16_reflected_zbc(s1[3], c256, s3[3]); - s3[4] = fold_16_reflected_zbc(s1[4], c256, s3[4]); - s3[5] = fold_16_reflected_zbc(s1[5], c256, s3[5]); - s3[6] = fold_16_reflected_zbc(s1[6], c256, s3[6]); - s3[7] = fold_16_reflected_zbc(s1[7], c256, s3[7]); - - s3[0] = fold_16_reflected_zbc(s0[0], c384, s3[0]); - s3[1] = fold_16_reflected_zbc(s0[1], c384, s3[1]); - s3[2] = fold_16_reflected_zbc(s0[2], c384, s3[2]); - s3[3] = fold_16_reflected_zbc(s0[3], c384, s3[3]); - s3[4] = fold_16_reflected_zbc(s0[4], c384, s3[4]); - s3[5] = fold_16_reflected_zbc(s0[5], c384, s3[5]); - s3[6] = fold_16_reflected_zbc(s0[6], c384, s3[6]); - s3[7] = fold_16_reflected_zbc(s0[7], c384, s3[7]); - - for block in &blocks[aligned..] { - fold_block_128_reflected_zbc(&mut s3, block, coeff_128); - } + let mut i: usize = 4; + while i < aligned { + fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_512); + fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128_reflected_zbc(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128_reflected_zbc(&mut s3, &blocks[i.strict_add(3)], coeff_512); + i = i.strict_add(4); + } - finalize_lanes_width32_reflected_zbc(s3, keys) + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + s3[0] = fold_16_reflected_zbc(s2[0], c128, s3[0]); + s3[1] = fold_16_reflected_zbc(s2[1], c128, s3[1]); + s3[2] = fold_16_reflected_zbc(s2[2], c128, s3[2]); + s3[3] = fold_16_reflected_zbc(s2[3], c128, s3[3]); + s3[4] = fold_16_reflected_zbc(s2[4], c128, s3[4]); + s3[5] = fold_16_reflected_zbc(s2[5], c128, s3[5]); + s3[6] = fold_16_reflected_zbc(s2[6], c128, s3[6]); + s3[7] = fold_16_reflected_zbc(s2[7], c128, s3[7]); + + s3[0] = fold_16_reflected_zbc(s1[0], c256, s3[0]); + s3[1] = fold_16_reflected_zbc(s1[1], c256, s3[1]); + s3[2] = fold_16_reflected_zbc(s1[2], c256, s3[2]); + s3[3] = fold_16_reflected_zbc(s1[3], c256, s3[3]); + s3[4] = fold_16_reflected_zbc(s1[4], c256, s3[4]); + s3[5] = fold_16_reflected_zbc(s1[5], c256, s3[5]); + s3[6] = fold_16_reflected_zbc(s1[6], c256, s3[6]); + s3[7] = fold_16_reflected_zbc(s1[7], c256, s3[7]); + + s3[0] = fold_16_reflected_zbc(s0[0], c384, s3[0]); + s3[1] = fold_16_reflected_zbc(s0[1], c384, s3[1]); + s3[2] = fold_16_reflected_zbc(s0[2], c384, s3[2]); + s3[3] = fold_16_reflected_zbc(s0[3], c384, s3[3]); + s3[4] = fold_16_reflected_zbc(s0[4], c384, s3[4]); + s3[5] = fold_16_reflected_zbc(s0[5], c384, s3[5]); + s3[6] = fold_16_reflected_zbc(s0[6], c384, s3[6]); + s3[7] = fold_16_reflected_zbc(s0[7], c384, s3[7]); + + for block in &blocks[aligned..] { + fold_block_128_reflected_zbc(&mut s3, block, coeff_128); } + + finalize_lanes_width32_reflected_zbc(s3, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc16_width32_zbc(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc16_width32_zbc(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -447,25 +398,25 @@ unsafe fn crc16_width32_zbc(mut state: u16, data: &[u8], keys: &[u64; 23], porta state = portable(state, left); let state32 = update_simd_zbc(state as u32, first, rest, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc16_width32_zbc_2way( +fn crc16_width32_zbc_2way( mut state: u16, data: &[u8], keys: &[u64; 23], stream: &super::keys::Width32StreamConstants, portable: fn(u16, &[u8]) -> u16, ) -> u16 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -473,25 +424,25 @@ unsafe fn crc16_width32_zbc_2way( } state = portable(state, left); let state32 = update_simd_zbc_2way(state as u32, middle, stream.fold_256b, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc16_width32_zbc_4way( +fn crc16_width32_zbc_4way( mut state: u16, data: &[u8], keys: &[u64; 23], stream: &super::keys::Width32StreamConstants, portable: fn(u16, &[u8]) -> u16, ) -> u16 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -499,7 +450,7 @@ unsafe fn crc16_width32_zbc_4way( } state = portable(state, left); let state32 = update_simd_zbc_4way(state as u32, middle, stream.fold_512b, &stream.combine_4way, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } @@ -513,10 +464,7 @@ unsafe fn crc16_width32_zbc_4way( /// Requires RISC-V `v` + `zvbc`. #[inline] #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. +fn mul64_zvbc(a: u64, b: u64) -> Simd { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -543,92 +491,68 @@ unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - mul64_zvbc(x.low_64(), coeff_high) ^ mul64_zvbc(x.high_64(), coeff_low) - } -} - -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_16_reflected_zvbc(x: Simd, coeff: (u64, u64), data_to_xor: Simd) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC + V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { data_to_xor ^ fold_16_zvbc(x, coeff) } +fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { + let (coeff_high, coeff_low) = coeff; + mul64_zvbc(x.low_64(), coeff_high) ^ mul64_zvbc(x.high_64(), coeff_low) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_width32_reflected_zvbc(x: Simd, high: u64, low: u64) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC + V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let clmul = mul64_zvbc(x.low_64(), low); - let shifted = Simd::new(0, x.high_64()); - let mut state = clmul ^ shifted; +fn fold_width32_reflected_zvbc(x: Simd, high: u64, low: u64) -> Simd { + let clmul = mul64_zvbc(x.low_64(), low); + let shifted = Simd::new(0, x.high_64()); + let mut state = clmul ^ shifted; - let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = mul64_zvbc(shifted_high, high); - state = clmul ^ masked; + let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = mul64_zvbc(shifted_high, high); + state = clmul ^ masked; - state - } + state } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn barrett_width32_reflected_zvbc(x: Simd, poly: u64, mu: u64) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = mul64_zvbc(x.low_64(), mu); - let l = mul64_zvbc(t1.low_64(), poly); - (x ^ l).high_64() as u32 - } +fn barrett_width32_reflected_zvbc(x: Simd, poly: u64, mu: u64) -> u32 { + let t1 = mul64_zvbc(x.low_64(), mu); + let l = mul64_zvbc(t1.low_64(), poly); + low_u32((x ^ l).high_64()) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut acc = Simd::new(hi[7], lo[7]); - acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), (keys[10], keys[9])); - acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), (keys[12], keys[11])); - acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), (keys[14], keys[13])); - acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), (keys[16], keys[15])); - acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), (keys[18], keys[17])); - acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), (keys[20], keys[19])); - acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), (keys[2], keys[1])); - - barrett_width32_reflected_zvbc(fold_width32_reflected_zvbc(acc, keys[6], keys[5]), keys[8], keys[7]) - } +fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], keys: &[u64; 23]) -> u32 { + let mut acc = Simd::new(hi[7], lo[7]); + acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), (keys[10], keys[9])); + acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), (keys[12], keys[11])); + acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), (keys[14], keys[13])); + acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), (keys[16], keys[15])); + acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), (keys[18], keys[17])); + acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), (keys[20], keys[19])); + acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), (keys[2], keys[1])); + + barrett_width32_reflected_zvbc(fold_width32_reflected_zvbc(acc, keys[6], keys[5]), keys[8], keys[7]) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_block_128_zvbc( +fn fold_block_128_zvbc( x_hi: &mut [u64; 8], x_lo: &mut [u64; 8], chunk_hi: &[u64; 8], @@ -636,15 +560,12 @@ unsafe fn fold_block_128_zvbc( coeff_low: u64, coeff_high: u64, ) { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { let mut offset = 0usize; while offset < 8 { - let remaining = 8 - offset; + let remaining = 8usize.strict_sub(offset); let vl: usize; asm!( "vsetvli {vl}, {avl}, e64, m1, ta, ma", @@ -684,174 +605,162 @@ unsafe fn fold_block_128_zvbc( } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (mut x_hi, mut x_lo) = load_block_split(first); - x_lo[0] ^= state as u64; +fn update_simd_zvbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { + let (mut x_hi, mut x_lo) = load_block_split(first); + x_lo[0] ^= state as u64; - let coeff_low = keys[3]; - let coeff_high = keys[4]; + let coeff_low = keys[3]; + let coeff_high = keys[4]; - for block in rest { - let (chunk_hi, chunk_lo) = load_block_split(block); - fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); - } - - fold_tail_zvbc(x_hi, x_lo, keys) + for block in rest { + let (chunk_hi, chunk_lo) = load_block_split(block); + fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); } + + fold_tail_zvbc(x_hi, x_lo, keys) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); +fn update_simd_zvbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { + debug_assert!(!blocks.is_empty()); - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, keys); - } - - let even = blocks.len() & !1usize; + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zvbc(state, first, rest, keys); + } - let coeff_256_low = fold_256b.1; - let coeff_256_high = fold_256b.0; - let coeff_128_low = keys[3]; - let coeff_128_high = keys[4]; + let even = blocks.len() & !1usize; - let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); + let coeff_256_low = fold_256b.1; + let coeff_256_high = fold_256b.0; + let coeff_128_low = keys[3]; + let coeff_128_high = keys[4]; - // Inject CRC into stream 0. - s0_lo[0] ^= state as u64; + let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); + let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); - let mut i: usize = 2; - while i < even { - let (b0_hi, b0_lo) = load_block_split(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); - i = i.strict_add(2); - } + // Inject CRC into stream 0. + s0_lo[0] ^= state as u64; - // Merge: A·s0 ⊕ s1 (A = shift by 128B). - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, coeff_128_low, coeff_128_high); - - if even != blocks.len() { - let (tail_hi, tail_lo) = load_block_split(&blocks[even]); - fold_block_128_zvbc( - &mut s0_hi, - &mut s0_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } + let mut i: usize = 2; + while i < even { + let (b0_hi, b0_lo) = load_block_split(&blocks[i]); + let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); + i = i.strict_add(2); + } - fold_tail_zvbc(s0_hi, s0_lo, keys) + // Merge: A·s0 ⊕ s1 (A = shift by 128B). + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, coeff_128_low, coeff_128_high); + + if even != blocks.len() { + let (tail_hi, tail_lo) = load_block_split(&blocks[even]); + fold_block_128_zvbc( + &mut s0_hi, + &mut s0_lo, + &tail_hi, + &tail_lo, + coeff_128_low, + coeff_128_high, + ); } + + fold_tail_zvbc(s0_hi, s0_lo, keys) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_4way( +fn update_simd_zvbc_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); + debug_assert!(!blocks.is_empty()); - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, keys); - } - - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512_low = fold_512b.1; - let coeff_512_high = fold_512b.0; - let coeff_128_low = keys[3]; - let coeff_128_high = keys[4]; - - let c384_low = combine[0].1; - let c384_high = combine[0].0; - let c256_low = combine[1].1; - let c256_high = combine[1].0; - let c128_low = combine[2].1; - let c128_high = combine[2].0; - - let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); - let (mut s2_hi, mut s2_lo) = load_block_split(&blocks[2]); - let (mut s3_hi, mut s3_lo) = load_block_split(&blocks[3]); - - // Inject CRC into stream 0. - s0_lo[0] ^= state as u64; - - let mut i: usize = 4; - while i < aligned { - let (b0_hi, b0_lo) = load_block_split(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); - let (b2_hi, b2_lo) = load_block_split(&blocks[i.strict_add(2)]); - let (b3_hi, b3_lo) = load_block_split(&blocks[i.strict_add(3)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); - i = i.strict_add(4); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zvbc(state, first, rest, keys); + } - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &s3_hi, &s3_lo, c128_low, c128_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &s2_hi, &s2_lo, c256_low, c256_high); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, c384_low, c384_high); - - for block in &blocks[aligned..] { - let (tail_hi, tail_lo) = load_block_split(block); - fold_block_128_zvbc( - &mut s0_hi, - &mut s0_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } + let aligned = blocks.len().strict_div(4).strict_mul(4); + + let coeff_512_low = fold_512b.1; + let coeff_512_high = fold_512b.0; + let coeff_128_low = keys[3]; + let coeff_128_high = keys[4]; + + let c384_low = combine[0].1; + let c384_high = combine[0].0; + let c256_low = combine[1].1; + let c256_high = combine[1].0; + let c128_low = combine[2].1; + let c128_high = combine[2].0; + + let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); + let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); + let (mut s2_hi, mut s2_lo) = load_block_split(&blocks[2]); + let (mut s3_hi, mut s3_lo) = load_block_split(&blocks[3]); + + // Inject CRC into stream 0. + s0_lo[0] ^= state as u64; + + let mut i: usize = 4; + while i < aligned { + let (b0_hi, b0_lo) = load_block_split(&blocks[i]); + let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); + let (b2_hi, b2_lo) = load_block_split(&blocks[i.strict_add(2)]); + let (b3_hi, b3_lo) = load_block_split(&blocks[i.strict_add(3)]); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); + i = i.strict_add(4); + } - fold_tail_zvbc(s0_hi, s0_lo, keys) + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &s3_hi, &s3_lo, c128_low, c128_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &s2_hi, &s2_lo, c256_low, c256_high); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, c384_low, c384_high); + + for block in &blocks[aligned..] { + let (tail_hi, tail_lo) = load_block_split(block); + fold_block_128_zvbc( + &mut s0_hi, + &mut s0_lo, + &tail_hi, + &tail_lo, + coeff_128_low, + coeff_128_high, + ); } + + fold_tail_zvbc(s0_hi, s0_lo, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc16_width32_zvbc(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc16_width32_zvbc(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -860,25 +769,25 @@ unsafe fn crc16_width32_zvbc(mut state: u16, data: &[u8], keys: &[u64; 23], port state = portable(state, left); let state32 = update_simd_zvbc(state as u32, first, rest, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc16_width32_zvbc_2way( +fn crc16_width32_zvbc_2way( mut state: u16, data: &[u8], keys: &[u64; 23], stream: &super::keys::Width32StreamConstants, portable: fn(u16, &[u8]) -> u16, ) -> u16 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -887,25 +796,25 @@ unsafe fn crc16_width32_zvbc_2way( state = portable(state, left); let state32 = update_simd_zvbc_2way(state as u32, middle, stream.fold_256b, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc16_width32_zvbc_4way( +fn crc16_width32_zvbc_4way( mut state: u16, data: &[u8], keys: &[u64; 23], stream: &super::keys::Width32StreamConstants, portable: fn(u16, &[u8]) -> u16, ) -> u16 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -914,7 +823,7 @@ unsafe fn crc16_width32_zvbc_4way( state = portable(state, left); let state32 = update_simd_zvbc_4way(state as u32, middle, stream.fold_512b, &stream.combine_4way, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } @@ -922,12 +831,8 @@ unsafe fn crc16_width32_zvbc_4way( // Public Safe Kernels /// CRC-16/CCITT Zbc kernel. -/// -/// # Safety -/// -/// Dispatcher verifies Zbc before selecting this kernel. #[inline] -pub fn crc16_ccitt_zbc_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_zbc_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc16_width32_zbc( @@ -940,12 +845,8 @@ pub fn crc16_ccitt_zbc_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/CCITT Zvbc kernel. -/// -/// # Safety -/// -/// Dispatcher verifies Zvbc before selecting this kernel. #[inline] -pub fn crc16_ccitt_zvbc_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_zvbc_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. unsafe { crc16_width32_zvbc( @@ -958,29 +859,21 @@ pub fn crc16_ccitt_zvbc_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/IBM Zbc kernel. -/// -/// # Safety -/// -/// Dispatcher verifies Zbc before selecting this kernel. #[inline] -pub fn crc16_ibm_zbc_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_zbc_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc16_width32_zbc(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } /// CRC-16/IBM Zvbc kernel. -/// -/// # Safety -/// -/// Dispatcher verifies Zvbc before selecting this kernel. #[inline] -pub fn crc16_ibm_zvbc_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_zvbc_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. unsafe { crc16_width32_zvbc(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } #[inline] -pub fn crc16_ccitt_zbc_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_zbc_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc16_width32_zbc_2way( @@ -994,7 +887,7 @@ pub fn crc16_ccitt_zbc_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_zbc_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_zbc_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc16_width32_zbc_4way( @@ -1008,7 +901,7 @@ pub fn crc16_ccitt_zbc_4way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_zvbc_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_zvbc_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. unsafe { crc16_width32_zvbc_2way( @@ -1022,7 +915,7 @@ pub fn crc16_ccitt_zvbc_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_zvbc_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_zvbc_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. unsafe { crc16_width32_zvbc_4way( @@ -1036,7 +929,7 @@ pub fn crc16_ccitt_zvbc_4way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_zbc_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_zbc_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc16_width32_zbc_2way( @@ -1050,7 +943,7 @@ pub fn crc16_ibm_zbc_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_zbc_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_zbc_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc16_width32_zbc_4way( @@ -1064,7 +957,7 @@ pub fn crc16_ibm_zbc_4way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_zvbc_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_zvbc_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. unsafe { crc16_width32_zvbc_2way( @@ -1078,7 +971,7 @@ pub fn crc16_ibm_zvbc_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_zvbc_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_zvbc_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. unsafe { crc16_width32_zvbc_4way( diff --git a/src/checksum/crc16/s390x.rs b/src/checksum/crc16/s390x.rs index fac679a1..8bc41a39 100644 --- a/src/checksum/crc16/s390x.rs +++ b/src/checksum/crc16/s390x.rs @@ -8,18 +8,15 @@ //! Uses `unsafe` for s390x vector + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, - mem::MaybeUninit, ops::{BitAnd, BitXor, BitXorAssign}, simd::i64x2, }; +use crate::checksum::common::{low_u16, low_u32}; + use super::keys::{ CRC16_CCITT_KEYS_REFLECTED, CRC16_CCITT_STREAM_REFLECTED, CRC16_IBM_KEYS_REFLECTED, CRC16_IBM_STREAM_REFLECTED, }; @@ -79,8 +76,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn vgfm(a: i64x2, b: i64x2) -> i64x2 { + fn vgfm(a: i64x2, b: i64x2) -> i64x2 { // SAFETY: Caller guarantees: // 1. VECTOR target features are available (dispatch check). // 2. All SIMD operations are pure register computations after loads. @@ -98,76 +98,66 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let va = Self::new(0, a); - let vb = Self::new(0, b); - Self(Self::vgfm(va.0, vb.0)) - } + fn mul64(a: u64, b: u64) -> Self { + let va = Self::new(0, a); + let vb = Self::new(0, b); + Self(Self::vgfm(va.0, vb.0)) } /// Fold 16 bytes (reflected width32 folding primitive): /// `self.low ⊗ coeff.high ⊕ self.high ⊗ coeff.low`. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_16(self, coeff: Self) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - // Like VPMSUMD: VGFM performs a per-lane carryless multiply and XORs the lane products. - // The folding primitive needs cross terms, so swap coefficient lanes. - Self(Self::vgfm(self.0, coeff.swap_lanes().0)) - } + fn fold_16(self, coeff: Self) -> Self { + Self(Self::vgfm(self.0, coeff.swap_lanes().0)) } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { data_to_xor ^ self.fold_16(coeff) } + fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { + data_to_xor ^ self.fold_16(coeff) } /// Fold 16 bytes down to the "width32" reduction state (reflected mode). #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_width32_reflected(self, high: u64, low: u64) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR + VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let clmul = Self::mul64(self.low_64(), low); - let shifted = Self::new(0, self.high_64()); - let mut state = clmul ^ shifted; - - let mask2 = Self::new(0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_0000_0000); - let masked = state & mask2; - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = Self::mul64(shifted_high, high); - state = clmul ^ masked; - - state - } + fn fold_width32_reflected(self, high: u64, low: u64) -> Self { + let clmul = Self::mul64(self.low_64(), low); + let shifted = Self::new(0, self.high_64()); + let mut state = clmul ^ shifted; + + let mask2 = Self::new(0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_0000_0000); + let masked = state & mask2; + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = Self::mul64(shifted_high, high); + state = clmul ^ masked; + + state } /// Barrett reduction for reflected width32; returns the updated CRC state. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let t1 = Self::mul64(self.low_64(), mu); - let l = Self::mul64(t1.low_64(), poly); - (self ^ l).high_64() as u32 - } + fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { + let t1 = Self::mul64(self.low_64(), mu); + let l = Self::mul64(t1.low_64(), poly); + low_u32((self ^ l).high_64()) } } @@ -175,239 +165,215 @@ impl Simd { #[inline(always)] fn load_block(block: &Block) -> [Simd; 8] { - let mut out = MaybeUninit::<[Simd; 8]>::uninit(); - let base = out.as_mut_ptr().cast::(); - - let mut i = 0; - while i < 8 { - let low = u64::from_le(block[i * 2]); - let high = u64::from_le(block[i * 2 + 1]); - // SAFETY: `base` points to a `[Simd; 8]` buffer and `i` is in-bounds. - unsafe { - base.add(i).write(Simd::new(high, low)); - } - i = i.strict_add(1); + let mut out = [Simd::new(0, 0); 8]; + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new(u64::from_le(high), u64::from_le(low)); } - - // SAFETY: all 8 elements are initialized above. - unsafe { out.assume_init() } + out } // Folding helpers #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let mut res = x[7]; - res = x[0].fold_16_reflected(Simd::new(keys[10], keys[9]), res); - res = x[1].fold_16_reflected(Simd::new(keys[12], keys[11]), res); - res = x[2].fold_16_reflected(Simd::new(keys[14], keys[13]), res); - res = x[3].fold_16_reflected(Simd::new(keys[16], keys[15]), res); - res = x[4].fold_16_reflected(Simd::new(keys[18], keys[17]), res); - res = x[5].fold_16_reflected(Simd::new(keys[20], keys[19]), res); - res = x[6].fold_16_reflected(Simd::new(keys[2], keys[1]), res); - - res = res.fold_width32_reflected(keys[6], keys[5]); - res.barrett_width32_reflected(keys[8], keys[7]) - } +fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u32 { + let mut res = x[7]; + res = x[0].fold_16_reflected(Simd::new(keys[10], keys[9]), res); + res = x[1].fold_16_reflected(Simd::new(keys[12], keys[11]), res); + res = x[2].fold_16_reflected(Simd::new(keys[14], keys[13]), res); + res = x[3].fold_16_reflected(Simd::new(keys[16], keys[15]), res); + res = x[4].fold_16_reflected(Simd::new(keys[18], keys[17]), res); + res = x[5].fold_16_reflected(Simd::new(keys[20], keys[19]), res); + res = x[6].fold_16_reflected(Simd::new(keys[2], keys[1]), res); + + res = res.fold_width32_reflected(keys[6], keys[5]); + res.barrett_width32_reflected(keys[8], keys[7]) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_width32_reflected(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let mut x = load_block(first); - - x[0] ^= Simd::new(0, state as u64); - - let coeff_128b = Simd::new(keys[4], keys[3]); - for block in rest { - let chunk = load_block(block); - x[0] = x[0].fold_16_reflected(coeff_128b, chunk[0]); - x[1] = x[1].fold_16_reflected(coeff_128b, chunk[1]); - x[2] = x[2].fold_16_reflected(coeff_128b, chunk[2]); - x[3] = x[3].fold_16_reflected(coeff_128b, chunk[3]); - x[4] = x[4].fold_16_reflected(coeff_128b, chunk[4]); - x[5] = x[5].fold_16_reflected(coeff_128b, chunk[5]); - x[6] = x[6].fold_16_reflected(coeff_128b, chunk[6]); - x[7] = x[7].fold_16_reflected(coeff_128b, chunk[7]); - } +fn update_simd_width32_reflected(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { + let mut x = load_block(first); - finalize_lanes_width32_reflected(x, keys) + x[0] ^= Simd::new(0, state as u64); + + let coeff_128b = Simd::new(keys[4], keys[3]); + for block in rest { + let chunk = load_block(block); + x[0] = x[0].fold_16_reflected(coeff_128b, chunk[0]); + x[1] = x[1].fold_16_reflected(coeff_128b, chunk[1]); + x[2] = x[2].fold_16_reflected(coeff_128b, chunk[2]); + x[3] = x[3].fold_16_reflected(coeff_128b, chunk[3]); + x[4] = x[4].fold_16_reflected(coeff_128b, chunk[4]); + x[5] = x[5].fold_16_reflected(coeff_128b, chunk[5]); + x[6] = x[6].fold_16_reflected(coeff_128b, chunk[6]); + x[7] = x[7].fold_16_reflected(coeff_128b, chunk[7]); } + + finalize_lanes_width32_reflected(x, keys) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn fold_block_128_reflected(x: &mut [Simd; 8], block: &Block, coeff: Simd) { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let chunk = load_block(block); - x[0] = x[0].fold_16_reflected(coeff, chunk[0]); - x[1] = x[1].fold_16_reflected(coeff, chunk[1]); - x[2] = x[2].fold_16_reflected(coeff, chunk[2]); - x[3] = x[3].fold_16_reflected(coeff, chunk[3]); - x[4] = x[4].fold_16_reflected(coeff, chunk[4]); - x[5] = x[5].fold_16_reflected(coeff, chunk[5]); - x[6] = x[6].fold_16_reflected(coeff, chunk[6]); - x[7] = x[7].fold_16_reflected(coeff, chunk[7]); - } +fn fold_block_128_reflected(x: &mut [Simd; 8], block: &Block, coeff: Simd) { + let chunk = load_block(block); + x[0] = x[0].fold_16_reflected(coeff, chunk[0]); + x[1] = x[1].fold_16_reflected(coeff, chunk[1]); + x[2] = x[2].fold_16_reflected(coeff, chunk[2]); + x[3] = x[3].fold_16_reflected(coeff, chunk[3]); + x[4] = x[4].fold_16_reflected(coeff, chunk[4]); + x[5] = x[5].fold_16_reflected(coeff, chunk[5]); + x[6] = x[6].fold_16_reflected(coeff, chunk[6]); + x[7] = x[7].fold_16_reflected(coeff, chunk[7]); } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_width32_reflected_2way( - state: u32, - blocks: &[Block], - fold_256b: (u64, u64), - keys: &[u64; 23], -) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - debug_assert!(!blocks.is_empty()); +fn update_simd_width32_reflected_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { + debug_assert!(!blocks.is_empty()); - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_width32_reflected(state, first, rest, keys); - } + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_width32_reflected(state, first, rest, keys); + } - let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); - let coeff_128 = Simd::new(keys[4], keys[3]); + let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); + let coeff_128 = Simd::new(keys[4], keys[3]); - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, state as u64); - let mut i: usize = 2; - let even = blocks.len() & !1usize; - while i < even { - fold_block_128_reflected(&mut s0, &blocks[i], coeff_256); - fold_block_128_reflected(&mut s1, &blocks[i.strict_add(1)], coeff_256); - i = i.strict_add(2); - } - - // Merge: A·s0 ⊕ s1. - let mut combined = s1; - combined[0] = s0[0].fold_16_reflected(coeff_128, combined[0]); - combined[1] = s0[1].fold_16_reflected(coeff_128, combined[1]); - combined[2] = s0[2].fold_16_reflected(coeff_128, combined[2]); - combined[3] = s0[3].fold_16_reflected(coeff_128, combined[3]); - combined[4] = s0[4].fold_16_reflected(coeff_128, combined[4]); - combined[5] = s0[5].fold_16_reflected(coeff_128, combined[5]); - combined[6] = s0[6].fold_16_reflected(coeff_128, combined[6]); - combined[7] = s0[7].fold_16_reflected(coeff_128, combined[7]); - - if even != blocks.len() { - fold_block_128_reflected(&mut combined, &blocks[even], coeff_128); - } + let mut i: usize = 2; + let even = blocks.len() & !1usize; + while i < even { + fold_block_128_reflected(&mut s0, &blocks[i], coeff_256); + fold_block_128_reflected(&mut s1, &blocks[i.strict_add(1)], coeff_256); + i = i.strict_add(2); + } - finalize_lanes_width32_reflected(combined, keys) + // Merge: A·s0 ⊕ s1. + let mut combined = s1; + combined[0] = s0[0].fold_16_reflected(coeff_128, combined[0]); + combined[1] = s0[1].fold_16_reflected(coeff_128, combined[1]); + combined[2] = s0[2].fold_16_reflected(coeff_128, combined[2]); + combined[3] = s0[3].fold_16_reflected(coeff_128, combined[3]); + combined[4] = s0[4].fold_16_reflected(coeff_128, combined[4]); + combined[5] = s0[5].fold_16_reflected(coeff_128, combined[5]); + combined[6] = s0[6].fold_16_reflected(coeff_128, combined[6]); + combined[7] = s0[7].fold_16_reflected(coeff_128, combined[7]); + + if even != blocks.len() { + fold_block_128_reflected(&mut combined, &blocks[even], coeff_128); } + + finalize_lanes_width32_reflected(combined, keys) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_width32_reflected_4way( +fn update_simd_width32_reflected_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - debug_assert!(!blocks.is_empty()); + debug_assert!(!blocks.is_empty()); - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_width32_reflected(state, first, rest, keys); - } - - let aligned = blocks.len().strict_div(4).strict_mul(4); + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_width32_reflected(state, first, rest, keys); + } - let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); - let coeff_128 = Simd::new(keys[4], keys[3]); - let c384 = Simd::new(combine[0].0, combine[0].1); - let c256 = Simd::new(combine[1].0, combine[1].1); - let c128 = Simd::new(combine[2].0, combine[2].1); + let aligned = blocks.len().strict_div(4).strict_mul(4); - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - let mut s2 = load_block(&blocks[2]); - let mut s3 = load_block(&blocks[3]); + let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); + let coeff_128 = Simd::new(keys[4], keys[3]); + let c384 = Simd::new(combine[0].0, combine[0].1); + let c256 = Simd::new(combine[1].0, combine[1].1); + let c128 = Simd::new(combine[2].0, combine[2].1); - s0[0] ^= Simd::new(0, state as u64); + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + let mut s2 = load_block(&blocks[2]); + let mut s3 = load_block(&blocks[3]); - let mut i: usize = 4; - while i < aligned { - fold_block_128_reflected(&mut s0, &blocks[i], coeff_512); - fold_block_128_reflected(&mut s1, &blocks[i.strict_add(1)], coeff_512); - fold_block_128_reflected(&mut s2, &blocks[i.strict_add(2)], coeff_512); - fold_block_128_reflected(&mut s3, &blocks[i.strict_add(3)], coeff_512); - i = i.strict_add(4); - } + s0[0] ^= Simd::new(0, state as u64); - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - let mut acc = s3; - acc[0] = s2[0].fold_16_reflected(c128, acc[0]); - acc[1] = s2[1].fold_16_reflected(c128, acc[1]); - acc[2] = s2[2].fold_16_reflected(c128, acc[2]); - acc[3] = s2[3].fold_16_reflected(c128, acc[3]); - acc[4] = s2[4].fold_16_reflected(c128, acc[4]); - acc[5] = s2[5].fold_16_reflected(c128, acc[5]); - acc[6] = s2[6].fold_16_reflected(c128, acc[6]); - acc[7] = s2[7].fold_16_reflected(c128, acc[7]); - - acc[0] = s1[0].fold_16_reflected(c256, acc[0]); - acc[1] = s1[1].fold_16_reflected(c256, acc[1]); - acc[2] = s1[2].fold_16_reflected(c256, acc[2]); - acc[3] = s1[3].fold_16_reflected(c256, acc[3]); - acc[4] = s1[4].fold_16_reflected(c256, acc[4]); - acc[5] = s1[5].fold_16_reflected(c256, acc[5]); - acc[6] = s1[6].fold_16_reflected(c256, acc[6]); - acc[7] = s1[7].fold_16_reflected(c256, acc[7]); - - acc[0] = s0[0].fold_16_reflected(c384, acc[0]); - acc[1] = s0[1].fold_16_reflected(c384, acc[1]); - acc[2] = s0[2].fold_16_reflected(c384, acc[2]); - acc[3] = s0[3].fold_16_reflected(c384, acc[3]); - acc[4] = s0[4].fold_16_reflected(c384, acc[4]); - acc[5] = s0[5].fold_16_reflected(c384, acc[5]); - acc[6] = s0[6].fold_16_reflected(c384, acc[6]); - acc[7] = s0[7].fold_16_reflected(c384, acc[7]); - - for block in &blocks[aligned..] { - fold_block_128_reflected(&mut acc, block, coeff_128); - } + let mut i: usize = 4; + while i < aligned { + fold_block_128_reflected(&mut s0, &blocks[i], coeff_512); + fold_block_128_reflected(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128_reflected(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128_reflected(&mut s3, &blocks[i.strict_add(3)], coeff_512); + i = i.strict_add(4); + } - finalize_lanes_width32_reflected(acc, keys) + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + let mut acc = s3; + acc[0] = s2[0].fold_16_reflected(c128, acc[0]); + acc[1] = s2[1].fold_16_reflected(c128, acc[1]); + acc[2] = s2[2].fold_16_reflected(c128, acc[2]); + acc[3] = s2[3].fold_16_reflected(c128, acc[3]); + acc[4] = s2[4].fold_16_reflected(c128, acc[4]); + acc[5] = s2[5].fold_16_reflected(c128, acc[5]); + acc[6] = s2[6].fold_16_reflected(c128, acc[6]); + acc[7] = s2[7].fold_16_reflected(c128, acc[7]); + + acc[0] = s1[0].fold_16_reflected(c256, acc[0]); + acc[1] = s1[1].fold_16_reflected(c256, acc[1]); + acc[2] = s1[2].fold_16_reflected(c256, acc[2]); + acc[3] = s1[3].fold_16_reflected(c256, acc[3]); + acc[4] = s1[4].fold_16_reflected(c256, acc[4]); + acc[5] = s1[5].fold_16_reflected(c256, acc[5]); + acc[6] = s1[6].fold_16_reflected(c256, acc[6]); + acc[7] = s1[7].fold_16_reflected(c256, acc[7]); + + acc[0] = s0[0].fold_16_reflected(c384, acc[0]); + acc[1] = s0[1].fold_16_reflected(c384, acc[1]); + acc[2] = s0[2].fold_16_reflected(c384, acc[2]); + acc[3] = s0[3].fold_16_reflected(c384, acc[3]); + acc[4] = s0[4].fold_16_reflected(c384, acc[4]); + acc[5] = s0[5].fold_16_reflected(c384, acc[5]); + acc[6] = s0[6].fold_16_reflected(c384, acc[6]); + acc[7] = s0[7].fold_16_reflected(c384, acc[7]); + + for block in &blocks[aligned..] { + fold_block_128_reflected(&mut acc, block, coeff_128); } + + finalize_lanes_width32_reflected(acc, keys) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc16_width32_vgfm(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. +fn crc16_width32_vgfm(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -416,23 +382,25 @@ unsafe fn crc16_width32_vgfm(mut state: u16, data: &[u8], keys: &[u64; 23], port state = portable(state, left); let state32 = update_simd_width32_reflected(state as u32, first, rest, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc16_width32_vgfm_2way( +fn crc16_width32_vgfm_2way( mut state: u16, data: &[u8], keys: &[u64; 23], stream: &super::keys::Width32StreamConstants, portable: fn(u16, &[u8]) -> u16, ) -> u16 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -441,23 +409,25 @@ unsafe fn crc16_width32_vgfm_2way( state = portable(state, left); let state32 = update_simd_width32_reflected_2way(state as u32, middle, stream.fold_256b, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc16_width32_vgfm_4way( +fn crc16_width32_vgfm_4way( mut state: u16, data: &[u8], keys: &[u64; 23], stream: &super::keys::Width32StreamConstants, portable: fn(u16, &[u8]) -> u16, ) -> u16 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -467,7 +437,7 @@ unsafe fn crc16_width32_vgfm_4way( state = portable(state, left); let state32 = update_simd_width32_reflected_4way(state as u32, middle, stream.fold_512b, &stream.combine_4way, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } @@ -475,12 +445,8 @@ unsafe fn crc16_width32_vgfm_4way( // Public Safe Kernels /// CRC-16/CCITT VGFM kernel. -/// -/// # Safety -/// -/// Dispatcher verifies VECTOR facility before selecting this kernel. #[inline] -pub fn crc16_ccitt_vgfm_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vgfm_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc16_width32_vgfm( @@ -493,7 +459,7 @@ pub fn crc16_ccitt_vgfm_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_vgfm_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vgfm_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc16_width32_vgfm_2way( @@ -507,7 +473,7 @@ pub fn crc16_ccitt_vgfm_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_vgfm_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vgfm_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc16_width32_vgfm_4way( @@ -521,18 +487,14 @@ pub fn crc16_ccitt_vgfm_4way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/IBM VGFM kernel. -/// -/// # Safety -/// -/// Dispatcher verifies VECTOR facility before selecting this kernel. #[inline] -pub fn crc16_ibm_vgfm_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vgfm_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc16_width32_vgfm(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } #[inline] -pub fn crc16_ibm_vgfm_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vgfm_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc16_width32_vgfm_2way( @@ -546,7 +508,7 @@ pub fn crc16_ibm_vgfm_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_vgfm_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vgfm_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc16_width32_vgfm_4way( diff --git a/src/checksum/crc16/x86_64.rs b/src/checksum/crc16/x86_64.rs index 95b270c7..6b7c3665 100644 --- a/src/checksum/crc16/x86_64.rs +++ b/src/checksum/crc16/x86_64.rs @@ -6,10 +6,9 @@ //! //! # Safety //! -//! Uses `unsafe` for x86 SIMD intrinsics. Callers must ensure SSSE3 + PCLMULQDQ -//! are available before executing these kernels (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] +//! The baseline kernels require SSE2, SSSE3, and PCLMULQDQ. The wide kernels +//! additionally require AVX-512F/VL/BW/DQ and VPCLMULQDQ. The private safe +//! wrappers are installed only through capability-gated dispatcher tables. use core::{ arch::x86_64::*, @@ -25,6 +24,12 @@ use super::keys::{ #[derive(Copy, Clone)] struct Simd128(__m128i); +#[inline] +const fn low_u16(value: u32) -> u16 { + let [low0, low1, ..] = value.to_le_bytes(); + u16::from_le_bytes([low0, low1]) +} + impl BitXor for Simd128 { type Output = Self; @@ -43,44 +48,87 @@ impl BitXorAssign for Simd128 { } impl Simd128 { + /// Loads 16 bytes without requiring alignment. + /// + /// # Safety + /// + /// `ptr` must be valid to read 16 initialized bytes. The source may be + /// unaligned because the bytes are copied into aligned local storage. + #[inline] + unsafe fn load_unaligned(ptr: *const u8) -> Self { + let mut value = core::mem::MaybeUninit::::uninit(); + + // SAFETY: The caller guarantees a readable 16-byte source. `value` is an + // aligned, non-overlapping 16-byte destination, and every bit pattern is + // valid for the integer vector inside `Simd128`. + unsafe { + core::ptr::copy_nonoverlapping(ptr, value.as_mut_ptr().cast::(), 16); + value.assume_init() + } + } + + /// Creates a vector from its high and low 64-bit lanes. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn new(high: u64, low: u64) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. Self(_mm_set_epi64x(high.cast_signed(), low.cast_signed())) } + /// Shifts the vector right by eight bytes, filling the high bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn shift_right_8(self) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. Self(_mm_srli_si128::<8>(self.0)) } + /// Shifts the vector left by 12 bytes, filling the low bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn shift_left_12(self) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. Self(_mm_slli_si128::<12>(self.0)) } + /// Computes the bitwise AND of two vectors. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn and(self, mask: Self) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. Self(_mm_and_si128(self.0, mask.0)) } + /// Folds one reflected 16-byte lane and XORs the supplied input lane. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. let h = _mm_clmulepi64_si128::<0x10>(self.0, coeff.0); let l = _mm_clmulepi64_si128::<0x01>(self.0, coeff.0); Self(_mm_xor_si128(_mm_xor_si128(h, l), data_to_xor.0)) } - /// Fold 16 bytes down to the "width32" reduction state (reflected mode). + /// Folds a reflected CRC state from 128 bits to the width-32 reduction state. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_width32_reflected(self, high: u64, low: u64) -> Self { @@ -105,6 +153,11 @@ impl Simd128 { } } + /// Applies Barrett reduction and returns the low width-32 state. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { @@ -116,11 +169,16 @@ impl Simd128 { let xorred = _mm_xor_si128(self.0, clmul2); let hi = _mm_srli_si128::<8>(xorred); - _mm_cvtsi128_si64(hi) as u32 + _mm_cvtsi128_si32(hi).cast_unsigned() } } } +/// Combines eight folded lanes and applies width-32 Barrett reduction. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn finalize_lanes_width32_reflected(x: [Simd128; 8], keys: &[u64; 23]) -> u32 { @@ -140,6 +198,11 @@ unsafe fn finalize_lanes_width32_reflected(x: [Simd128; 8], keys: &[u64; 23]) -> } } +/// Folds one or more 128-byte blocks into a width-32 CRC state. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected( @@ -172,6 +235,11 @@ unsafe fn update_simd_width32_reflected( // PCLMULQDQ multi-stream (2/4/7/8-way, 128B blocks) +/// Folds one 128-byte block into eight parallel lanes. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_block_128_reflected(x: &mut [Simd128; 8], chunk: &[Simd128; 8], coeff: Simd128) { @@ -188,6 +256,11 @@ unsafe fn fold_block_128_reflected(x: &mut [Simd128; 8], chunk: &[Simd128; 8], c } } +/// Folds 128-byte blocks through two parallel PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_2way( @@ -222,7 +295,7 @@ unsafe fn update_simd_width32_reflected_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -268,6 +341,11 @@ unsafe fn update_simd_width32_reflected_2way( } } +/// Folds 128-byte blocks through four parallel PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_4way( @@ -308,7 +386,7 @@ unsafe fn update_simd_width32_reflected_4way( const DOUBLE_GROUP: usize = 8; // 2 × 4-way = 8 blocks = 1KB let mut i: usize = 4; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -332,7 +410,7 @@ unsafe fn update_simd_width32_reflected_4way( } // Handle remaining quads. - let quad_aligned = (blocks.len() / 4) * 4; + let quad_aligned = blocks.len().strict_sub(blocks.len().strict_rem(4)); while i < quad_aligned { fold_block_128_reflected(&mut s0, &blocks[i], coeff_512); fold_block_128_reflected(&mut s1, &blocks[i.strict_add(1)], coeff_512); @@ -378,6 +456,11 @@ unsafe fn update_simd_width32_reflected_4way( } } +/// Folds 128-byte blocks through seven parallel PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_7way( @@ -400,7 +483,7 @@ unsafe fn update_simd_width32_reflected_7way( return update_simd_width32_reflected(state, first, rest, keys); } - let aligned = (blocks.len() / 7) * 7; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(7)); let coeff_896 = Simd128::new(fold_896b.0, fold_896b.1); let coeff_128 = Simd128::new(keys[4], keys[3]); @@ -504,6 +587,11 @@ unsafe fn update_simd_width32_reflected_7way( } } +/// Folds 128-byte blocks through eight parallel PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_8way( @@ -526,7 +614,7 @@ unsafe fn update_simd_width32_reflected_8way( return update_simd_width32_reflected(state, first, rest, keys); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(8)); let coeff_1024 = Simd128::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd128::new(keys[4], keys[3]); @@ -642,6 +730,11 @@ unsafe fn update_simd_width32_reflected_8way( } } +/// Updates a CRC-16 value with a selected multi-stream PCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn crc16_width32_pclmul_stream( @@ -668,11 +761,16 @@ unsafe fn crc16_width32_pclmul_stream( 2 => update_simd_width32_reflected_2way(state as u32, middle, stream.fold_256b, keys), _ => update_simd_width32_reflected(state as u32, first, rest, keys), }; - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } +/// Updates a CRC-16 value with the single-stream PCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn crc16_width32_pclmul_small( @@ -694,26 +792,31 @@ unsafe fn crc16_width32_pclmul_small( let coeff_16b = Simd128::new(keys[2], keys[1]); - let mut x0 = Simd128(_mm_loadu_si128(buf as *const __m128i)); + let mut x0 = Simd128::load_unaligned(buf); x0 ^= Simd128::new(0, state as u64); buf = buf.add(16); len = len.strict_sub(16); while len >= 16 { - let chunk = Simd128(_mm_loadu_si128(buf as *const __m128i)); + let chunk = Simd128::load_unaligned(buf); x0 = x0.fold_16_reflected(coeff_16b, chunk); buf = buf.add(16); len = len.strict_sub(16); } let x0 = x0.fold_width32_reflected(keys[6], keys[5]); - state = x0.barrett_width32_reflected(keys[8], keys[7]) as u16; + state = low_u16(x0.barrett_width32_reflected(keys[8], keys[7])); let tail = core::slice::from_raw_parts(buf, len); portable(state, tail) } } +/// Updates a CRC-16 value with the baseline PCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn crc16_width32_pclmul(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { @@ -727,29 +830,74 @@ unsafe fn crc16_width32_pclmul(mut state: u16, data: &[u8], keys: &[u64; 23], po state = portable(state, left); let state32 = update_simd_width32_reflected(state as u32, first, rest, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } // AVX-512 VPCLMULQDQ Tier +/// Loads 64 bytes without requiring alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 64 initialized bytes. The source may be +/// unaligned because the bytes are copied into aligned local storage. +#[inline] +unsafe fn load_unaligned_512(ptr: *const u8) -> __m512i { + let mut value = core::mem::MaybeUninit::<__m512i>::uninit(); + + // SAFETY: The caller guarantees a readable 64-byte source. `value` is an + // aligned, non-overlapping 64-byte destination, and every bit pattern is + // valid for an integer vector. + unsafe { + core::ptr::copy_nonoverlapping(ptr, value.as_mut_ptr().cast::(), 64); + value.assume_init() + } +} + +/// Stores 64 bytes without requiring alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to write 64 bytes and must not overlap `value`. +#[inline] +unsafe fn store_unaligned_512(ptr: *mut u8, value: __m512i) { + // SAFETY: The caller guarantees a writable, non-overlapping 64-byte + // destination. `value` provides exactly 64 initialized source bytes. + unsafe { core::ptr::copy_nonoverlapping(core::ptr::from_ref(&value).cast::(), ptr, 64) } +} + +/// Multiplies the high lane of each 128-bit element in `a` by the low lane in `b`. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, AVX-512DQ, and +/// VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul10_vpclmul(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_clmulepi64_epi128(a, b, 0x10) } +/// Multiplies the low lane of each 128-bit element in `a` by the high lane in `b`. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, AVX-512DQ, and +/// VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul01_vpclmul(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_clmulepi64_epi128(a, b, 0x01) } +/// Folds four reflected 16-byte lanes and XORs their supplied input lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, AVX-512DQ, and +/// VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn fold_16_reflected_vpclmul(state: __m512i, coeff: __m512i, data: __m512i) -> __m512i { @@ -758,11 +906,14 @@ unsafe fn fold_16_reflected_vpclmul(state: __m512i, coeff: __m512i, data: __m512 unsafe { _mm512_ternarylogic_epi64(clmul10_vpclmul(state, coeff), clmul01_vpclmul(state, coeff), data, 0x96) } } +/// Broadcasts a pair of 64-bit coefficients across four 128-bit lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn broadcast_coeff_128b(high: u64, low: u64) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_set_epi64( high.cast_signed(), low.cast_signed(), @@ -775,14 +926,22 @@ unsafe fn broadcast_coeff_128b(high: u64, low: u64) -> __m512i { ) } +/// Places the width-32 CRC state in the low 32 bits of lane zero. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn state_mask_lane0(state: u32) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_set_epi64(0, 0, 0, 0, 0, 0, 0, state as i64) } +/// Folds one or more 128-byte blocks into a width-32 CRC state with VPCLMULQDQ. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul( @@ -795,25 +954,25 @@ unsafe fn update_simd_width32_reflected_vpclmul( // attribute. Pointer arithmetic: first has 8 Simd128 = 128 bytes, so ptr.add(64) is within // bounds. Each chunk in rest is likewise 128 bytes. unsafe { - let ptr = first.as_ptr() as *const u8; - let mut x0 = _mm512_loadu_si512(ptr as *const __m512i); - let mut x1 = _mm512_loadu_si512(ptr.add(64) as *const __m512i); + let ptr = first.as_ptr().cast::(); + let mut x0 = load_unaligned_512(ptr); + let mut x1 = load_unaligned_512(ptr.add(64)); x0 = _mm512_xor_si512(x0, state_mask_lane0(state)); let coeff_128b = broadcast_coeff_128b(keys[4], keys[3]); for chunk in rest { - let ptr = chunk.as_ptr() as *const u8; - let y0 = _mm512_loadu_si512(ptr as *const __m512i); - let y1 = _mm512_loadu_si512(ptr.add(64) as *const __m512i); + let ptr = chunk.as_ptr().cast::(); + let y0 = load_unaligned_512(ptr); + let y1 = load_unaligned_512(ptr.add(64)); x0 = fold_16_reflected_vpclmul(x0, coeff_128b, y0); x1 = fold_16_reflected_vpclmul(x1, coeff_128b, y1); } let mut lanes0 = [Simd128(_mm_setzero_si128()); 4]; let mut lanes1 = [Simd128(_mm_setzero_si128()); 4]; - _mm512_storeu_si512(lanes0.as_mut_ptr() as *mut __m512i, x0); - _mm512_storeu_si512(lanes1.as_mut_ptr() as *mut __m512i, x1); + store_unaligned_512(lanes0.as_mut_ptr().cast::(), x0); + store_unaligned_512(lanes1.as_mut_ptr().cast::(), x1); let x = [ lanes0[0], lanes0[1], lanes0[2], lanes0[3], lanes1[0], lanes1[1], lanes1[2], lanes1[3], @@ -823,6 +982,11 @@ unsafe fn update_simd_width32_reflected_vpclmul( } } +/// Broadcasts one pair of folding coefficients across four 128-bit lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { @@ -831,20 +995,27 @@ unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { unsafe { broadcast_coeff_128b(pair.0, pair.1) } } +/// Loads a 128-byte block as two unaligned 512-bit vectors. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn load_128b_block(block: &[Simd128; 8]) -> (__m512i, __m512i) { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] // attribute. block has 8 Simd128 = 128 bytes, so ptr.add(64) is within bounds. unsafe { - let ptr = block.as_ptr() as *const u8; - ( - _mm512_loadu_si512(ptr as *const __m512i), - _mm512_loadu_si512(ptr.add(64) as *const __m512i), - ) + let ptr = block.as_ptr().cast::(); + (load_unaligned_512(ptr), load_unaligned_512(ptr.add(64))) } } +/// Combines two four-lane VPCLMULQDQ states and applies width-32 reduction. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, SSE2, and PCLMULQDQ. #[inline] #[target_feature(enable = "avx512f")] unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i, keys: &[u64; 23]) -> u32 { @@ -871,6 +1042,11 @@ unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i, keys: &[u64; 23]) -> } } +/// Folds 128-byte blocks through two parallel VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_2way( @@ -906,7 +1082,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -962,6 +1138,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_2way( } } +/// Folds 128-byte blocks through four parallel VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_4way( @@ -1003,7 +1184,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_4way( const DOUBLE_GROUP: usize = 8; // 2 × 4-way = 8 blocks = 1KB let mut i: usize = 4; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -1049,7 +1230,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_4way( } // Handle remaining quads. - let quad_aligned = (blocks.len() / 4) * 4; + let quad_aligned = blocks.len().strict_sub(blocks.len().strict_rem(4)); while i < quad_aligned { let (y0, y1) = load_128b_block(&blocks[i]); x0_0 = fold_16_reflected_vpclmul(x0_0, coeff_512, y0); @@ -1087,6 +1268,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_4way( } } +/// Folds 128-byte blocks through seven parallel VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_7way( @@ -1110,7 +1296,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_7way( return update_simd_width32_reflected_vpclmul(state, first, rest, keys); } - let aligned = (blocks.len() / 7) * 7; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(7)); let (mut x0_0, mut x1_0) = load_128b_block(&blocks[0]); let (mut x0_1, mut x1_1) = load_128b_block(&blocks[1]); @@ -1194,6 +1380,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_7way( } } +/// Folds 128-byte blocks through eight parallel VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_8way( @@ -1217,7 +1408,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_8way( return update_simd_width32_reflected_vpclmul(state, first, rest, keys); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(8)); let (mut x0_0, mut x1_0) = load_128b_block(&blocks[0]); let (mut x0_1, mut x1_1) = load_128b_block(&blocks[1]); @@ -1309,6 +1500,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_8way( } } +/// Updates a CRC-16 value with a selected multi-stream VPCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn crc16_width32_vpclmul_stream( @@ -1341,11 +1537,16 @@ unsafe fn crc16_width32_vpclmul_stream( 2 => update_simd_width32_reflected_vpclmul_2way(state as u32, middle, stream.fold_256b, keys), _ => update_simd_width32_reflected_vpclmul(state as u32, first, rest, keys), }; - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } +/// Updates a CRC-16 value with the baseline VPCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn crc16_width32_vpclmul(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { @@ -1359,20 +1560,19 @@ unsafe fn crc16_width32_vpclmul(mut state: u16, data: &[u8], keys: &[u64; 23], p state = portable(state, left); let state32 = update_simd_width32_reflected_vpclmul(state as u32, first, rest, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } -// Public Safe Kernels (matching CRC-64 pure fn(u16, &[u8]) -> u16 signature) +// Private safe kernel adapters matching the dispatcher's function signature. /// CRC-16/CCITT PCLMULQDQ kernel. /// -/// # Safety -/// -/// Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying SSSE3 and +/// PCLMULQDQ support. #[inline] -pub fn crc16_ccitt_pclmul_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_pclmul_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul( @@ -1388,11 +1588,10 @@ pub fn crc16_ccitt_pclmul_safe(crc: u16, data: &[u8]) -> u16 { /// /// Optimized for inputs smaller than a folding block (128 bytes). /// -/// # Safety -/// -/// Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying SSSE3 and +/// PCLMULQDQ support. #[inline] -pub fn crc16_ccitt_pclmul_small_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_pclmul_small_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_small( @@ -1406,7 +1605,7 @@ pub fn crc16_ccitt_pclmul_small_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT PCLMULQDQ kernel (2-way multi-stream). #[inline] -pub fn crc16_ccitt_pclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_pclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1422,7 +1621,7 @@ pub fn crc16_ccitt_pclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT PCLMULQDQ kernel (4-way multi-stream). #[inline] -pub fn crc16_ccitt_pclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_pclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1438,7 +1637,7 @@ pub fn crc16_ccitt_pclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT PCLMULQDQ kernel (7-way multi-stream). #[inline] -pub fn crc16_ccitt_pclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_pclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1454,7 +1653,7 @@ pub fn crc16_ccitt_pclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT PCLMULQDQ kernel (8-way multi-stream). #[inline] -pub fn crc16_ccitt_pclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_pclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1470,11 +1669,10 @@ pub fn crc16_ccitt_pclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT VPCLMULQDQ kernel (AVX-512). /// -/// # Safety -/// -/// Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying VPCLMULQDQ +/// and AVX-512 support. #[inline] -pub fn crc16_ccitt_vpclmul_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpclmul_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul( @@ -1488,7 +1686,7 @@ pub fn crc16_ccitt_vpclmul_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT VPCLMULQDQ kernel (2-way multi-stream). #[inline] -pub fn crc16_ccitt_vpclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1504,7 +1702,7 @@ pub fn crc16_ccitt_vpclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT VPCLMULQDQ kernel (4-way multi-stream). #[inline] -pub fn crc16_ccitt_vpclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1520,7 +1718,7 @@ pub fn crc16_ccitt_vpclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT VPCLMULQDQ kernel (7-way multi-stream). #[inline] -pub fn crc16_ccitt_vpclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1536,7 +1734,7 @@ pub fn crc16_ccitt_vpclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT VPCLMULQDQ kernel (8-way multi-stream). #[inline] -pub fn crc16_ccitt_vpclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1552,11 +1750,10 @@ pub fn crc16_ccitt_vpclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM PCLMULQDQ kernel. /// -/// # Safety -/// -/// Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying SSSE3 and +/// PCLMULQDQ support. #[inline] -pub fn crc16_ibm_pclmul_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_pclmul_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } @@ -1565,18 +1762,17 @@ pub fn crc16_ibm_pclmul_safe(crc: u16, data: &[u8]) -> u16 { /// /// Optimized for inputs smaller than a folding block (128 bytes). /// -/// # Safety -/// -/// Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying SSSE3 and +/// PCLMULQDQ support. #[inline] -pub fn crc16_ibm_pclmul_small_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_pclmul_small_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_small(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } /// CRC-16/IBM PCLMULQDQ kernel (2-way multi-stream). #[inline] -pub fn crc16_ibm_pclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_pclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1592,7 +1788,7 @@ pub fn crc16_ibm_pclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM PCLMULQDQ kernel (4-way multi-stream). #[inline] -pub fn crc16_ibm_pclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_pclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1608,7 +1804,7 @@ pub fn crc16_ibm_pclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM PCLMULQDQ kernel (7-way multi-stream). #[inline] -pub fn crc16_ibm_pclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_pclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1624,7 +1820,7 @@ pub fn crc16_ibm_pclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM PCLMULQDQ kernel (8-way multi-stream). #[inline] -pub fn crc16_ibm_pclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_pclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1640,18 +1836,17 @@ pub fn crc16_ibm_pclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM VPCLMULQDQ kernel (AVX-512). /// -/// # Safety -/// -/// Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying VPCLMULQDQ +/// and AVX-512 support. #[inline] -pub fn crc16_ibm_vpclmul_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpclmul_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } /// CRC-16/IBM VPCLMULQDQ kernel (2-way multi-stream). #[inline] -pub fn crc16_ibm_vpclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1667,7 +1862,7 @@ pub fn crc16_ibm_vpclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM VPCLMULQDQ kernel (4-way multi-stream). #[inline] -pub fn crc16_ibm_vpclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1683,7 +1878,7 @@ pub fn crc16_ibm_vpclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM VPCLMULQDQ kernel (7-way multi-stream). #[inline] -pub fn crc16_ibm_vpclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1699,7 +1894,7 @@ pub fn crc16_ibm_vpclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM VPCLMULQDQ kernel (8-way multi-stream). #[inline] -pub fn crc16_ibm_vpclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1725,14 +1920,21 @@ mod tests { const OFFSETS: &[usize] = &[0, 1, 7, 15]; const STATES: &[u16] = &[0, 0x1d0f, 0xa5a5, u16::MAX]; + fn data() -> Vec { + (0u16..4111) + .map(|i| { + let [low, high] = i.to_le_bytes(); + low.wrapping_mul(59).wrapping_add(high) + }) + .collect() + } + fn assert_kernel(name: &str, kernel: fn(u16, &[u8]) -> u16, portable: fn(u16, &[u8]) -> u16) { - let input: Vec = (0..4111) - .map(|i| (i as u8).wrapping_mul(59).wrapping_add((i >> 8) as u8)) - .collect(); + let input = data(); for &state in STATES { for &offset in OFFSETS { for &len in LENS { - let slice = &input[offset..offset + len]; + let slice = &input[offset..offset.strict_add(len)]; assert_eq!( kernel(state, slice), portable(state, slice), diff --git a/src/checksum/crc24/aarch64.rs b/src/checksum/crc24/aarch64.rs index 294e96a7..360923a9 100644 --- a/src/checksum/crc24/aarch64.rs +++ b/src/checksum/crc24/aarch64.rs @@ -6,10 +6,8 @@ //! //! # Safety //! -//! Uses `unsafe` for ARM SIMD intrinsics. Callers must ensure PMULL is -//! available before executing these kernels (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] +//! Uses `unsafe` for ARM SIMD intrinsics. Callers must establish NEON and AES +//! (PMULL) support before executing the accelerated kernels. use core::{ arch::aarch64::*, @@ -45,33 +43,57 @@ impl BitXorAssign for Simd { } impl Simd { + /// Creates a vector from its high and low 64-bit lanes. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn new(high: u64, low: u64) -> Self { Self(vcombine_u8(vcreate_u8(low), vcreate_u8(high))) } + /// Loads 16 bytes from `ptr` without requiring alignment. + /// + /// # Safety + /// + /// The current CPU must support NEON, and `ptr` must address at least 16 initialized readable + /// bytes. #[inline] #[target_feature(enable = "neon")] unsafe fn load(ptr: *const u8) -> Self { - // SAFETY: Caller guarantees: - // 1. NEON target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + // SAFETY: The caller guarantees NEON support and 16 initialized readable bytes at `ptr`. unsafe { Self(vld1q_u8(ptr)) } } + /// Computes the bitwise AND of two vectors. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn and(self, mask: Self) -> Self { Self(vandq_u8(self.0, mask.0)) } + /// Shifts the vector right by eight bytes, filling the high bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn shift_right_8(self) -> Self { Self(vextq_u8(self.0, vdupq_n_u8(0), 8)) } + /// Moves the low 32-bit lane to the high 32-bit lane and clears the rest. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn shift_left_12(self) -> Self { @@ -80,13 +102,22 @@ impl Simd { Self(vreinterpretq_u8_u32(result)) } - /// Reverse bits within each byte (u8::reverse_bits), lane-wise. + /// Reverses the bits within each byte lane. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn bitrev_bytes(self) -> Self { Self(vrbitq_u8(self.0)) } + /// Multiplies the low polynomial lanes. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul00(self, other: Self) -> Self { @@ -95,6 +126,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Multiplies this vector's high polynomial lane by the other vector's low lane. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul01(self, other: Self) -> Self { @@ -103,6 +139,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Multiplies this vector's low polynomial lane by the other vector's high lane. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul10(self, other: Self) -> Self { @@ -111,6 +152,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Multiplies the high polynomial lanes. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul11(self, other: Self) -> Self { @@ -119,6 +165,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Folds one reflected 16-byte lane and XORs the supplied input lane. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { @@ -132,6 +183,11 @@ impl Simd { } } + /// Folds a reflected CRC state from 128 bits to the width-32 reduction state. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn fold_width32_reflected(self, high: u64, low: u64) -> Self { @@ -156,6 +212,11 @@ impl Simd { } } + /// Applies Barrett reduction and returns the low CRC-24 state in a `u32`. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { @@ -176,6 +237,11 @@ impl Simd { // 8-lane width32 update (128B blocks) +/// Reduces eight folded SIMD lanes to a reflected CRC-24 state. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u32 { @@ -197,6 +263,11 @@ unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u3 } } +/// Reverses the bits within every byte of a 128-byte block. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn bitrev_block(block: &[Simd; 8]) -> [Simd; 8] { @@ -217,6 +288,11 @@ unsafe fn bitrev_block(block: &[Simd; 8]) -> [Simd; 8] { } } +/// Bit-reverses and folds one 128-byte block into the current SIMD state. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn fold_block_128_width32_reflected_bitrev_bytes(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { @@ -244,6 +320,11 @@ unsafe fn fold_block_128_width32_reflected_bitrev_bytes(x: &mut [Simd; 8], chunk } } +/// Bit-reverses and folds 128-byte blocks through two independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_width32_reflected_bitrev_bytes_2way( state: u32, @@ -274,7 +355,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -321,6 +402,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_2way( } } +/// Bit-reverses and folds 128-byte blocks through three independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_width32_reflected_bitrev_bytes_3way( state: u32, @@ -359,7 +445,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_3way( const DOUBLE_GROUP: usize = 6; // 2 × 3-way = 6 blocks = 768B let mut i: usize = 3; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -381,7 +467,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_3way( } // Handle remaining triplets. - let triple_aligned = (blocks.len() / 3) * 3; + let triple_aligned = blocks.len().strict_div(3).strict_mul(3); while i < triple_aligned { fold_block_128_width32_reflected_bitrev_bytes(&mut s0, &blocks[i], coeff_384b); fold_block_128_width32_reflected_bitrev_bytes(&mut s1, &blocks[i.strict_add(1)], coeff_384b); @@ -419,6 +505,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_3way( } } +/// Bit-reverses and folds a reflected sequence of 128-byte blocks. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_width32_reflected_bitrev_bytes( @@ -471,6 +562,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes( // Single-stream kernel entry points +/// Computes reflected CRC-24 with single-lane PMULL folding for small buffers. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc24_width32_pmull_small(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { @@ -508,6 +604,11 @@ unsafe fn crc24_width32_pmull_small(mut state: u32, data: &[u8], keys: &[u64; 23 } } +/// Computes reflected CRC-24 with PMULL folding. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc24_width32_pmull(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { @@ -527,6 +628,11 @@ unsafe fn crc24_width32_pmull(mut state: u32, data: &[u8], keys: &[u64; 23]) -> } } +/// Computes reflected CRC-24 with two independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc24_width32_pmull_2way(mut state: u32, data: &[u8], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { @@ -549,6 +655,11 @@ unsafe fn crc24_width32_pmull_2way(mut state: u32, data: &[u8], fold_256b: (u64, } } +/// Computes reflected CRC-24 with three independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc24_width32_pmull_3way( @@ -573,17 +684,13 @@ unsafe fn crc24_width32_pmull_3way( } } -// Public Safe Kernel +// Safe kernel wrappers. /// CRC-24/OPENPGP PMULL kernel. -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc24_openpgp_pmull_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pmull_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. + // SAFETY: All callers establish PMULL support before invoking this private wrapper. state = unsafe { crc24_width32_pmull(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; from_reflected_state(state) } @@ -591,27 +698,19 @@ pub fn crc24_openpgp_pmull_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP PMULL small-buffer kernel. /// /// Optimized for inputs smaller than a folding block (128 bytes). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc24_openpgp_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. + // SAFETY: All callers establish PMULL support before invoking this private wrapper. state = unsafe { crc24_width32_pmull_small(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; from_reflected_state(state) } /// CRC-24/OPENPGP PMULL kernel (2-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc24_openpgp_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. + // SAFETY: All callers establish PMULL support before invoking this private wrapper. state = unsafe { crc24_width32_pmull_2way( state, @@ -624,14 +723,10 @@ pub fn crc24_openpgp_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { } /// CRC-24/OPENPGP PMULL kernel (3-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc24_openpgp_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. + // SAFETY: All callers establish PMULL support before invoking this private wrapper. state = unsafe { crc24_width32_pmull_3way( state, @@ -657,8 +752,11 @@ mod tests { const STATES: &[u32] = &[0, 0x00b7_04ce, 0x005a_a5a5, 0x00ff_ffff]; fn data() -> Vec { - (0..4111) - .map(|i| (i as u8).wrapping_mul(31).wrapping_add((i >> 8) as u8)) + (0u16..4111) + .map(|i| { + let [low, high] = i.to_le_bytes(); + low.wrapping_mul(31).wrapping_add(high) + }) .collect() } @@ -678,7 +776,7 @@ mod tests { for &state in STATES { for &offset in OFFSETS { for &len in LENS { - let slice = &input[offset..offset + len]; + let slice = &input[offset..offset.strict_add(len)]; assert_eq!( kernel(state, slice), super::super::portable::crc24_openpgp_slice8(state, slice), diff --git a/src/checksum/crc24/config.rs b/src/checksum/crc24/config.rs index ac14fdbe..462048d2 100644 --- a/src/checksum/crc24/config.rs +++ b/src/checksum/crc24/config.rs @@ -82,8 +82,16 @@ fn parse_force_env() -> Crc24Force { #[inline] #[must_use] -#[allow(unused_variables)] fn clamp_force_to_caps(requested: Crc24Force, caps: Caps) -> Crc24Force { + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" + )))] + let _ = caps; + match requested { Crc24Force::Auto | Crc24Force::Reference | Crc24Force::Portable => requested, Crc24Force::Clmul => { @@ -158,7 +166,7 @@ fn config(caps: Caps) -> Crc24Config { /// platform capabilities. #[inline] #[must_use] -pub fn get() -> Crc24Config { +pub(super) fn get() -> Crc24Config { #[cfg(feature = "std")] { use std::sync::OnceLock; diff --git a/src/checksum/crc24/kernels.rs b/src/checksum/crc24/kernels.rs index 12c4700a..6d7e022e 100644 --- a/src/checksum/crc24/kernels.rs +++ b/src/checksum/crc24/kernels.rs @@ -24,22 +24,22 @@ //! state back to the MSB-first OpenPGP representation. /// Reference (bitwise) kernel name. -pub use kernels::REFERENCE; +pub(in crate::checksum) use kernels::REFERENCE; use crate::checksum::common::kernels; /// Portable slice-by-8 kernel name. -pub const PORTABLE_SLICE8: &str = kernels::PORTABLE_SLICE8; +pub(in crate::checksum) const PORTABLE_SLICE8: &str = kernels::PORTABLE_SLICE8; // Kernel Name Tables and Functions (per architecture) #[cfg(target_arch = "x86_64")] -pub mod x86_64 { +pub(in crate::checksum) mod x86_64 { use super::super::x86_64 as arch; use crate::checksum::dispatchers::Crc24Fn; /// OpenPGP PCLMUL kernel. - pub const OPENPGP_PCLMUL: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_PCLMUL: [Crc24Fn; 5] = [ arch::crc24_openpgp_pclmul_safe, arch::crc24_openpgp_pclmul_2way_safe, arch::crc24_openpgp_pclmul_4way_safe, @@ -48,10 +48,10 @@ pub mod x86_64 { ]; /// OpenPGP PCLMUL small-buffer kernel. - pub const OPENPGP_PCLMUL_SMALL_KERNEL: Crc24Fn = arch::crc24_openpgp_pclmul_small_safe; + pub(in crate::checksum) const OPENPGP_PCLMUL_SMALL_KERNEL: Crc24Fn = arch::crc24_openpgp_pclmul_small_safe; /// OpenPGP VPCLMUL kernel. - pub const OPENPGP_VPCLMUL: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_VPCLMUL: [Crc24Fn; 5] = [ arch::crc24_openpgp_vpclmul_safe, arch::crc24_openpgp_vpclmul_2way_safe, arch::crc24_openpgp_vpclmul_4way_safe, @@ -61,12 +61,12 @@ pub mod x86_64 { } #[cfg(target_arch = "aarch64")] -pub mod aarch64 { +pub(in crate::checksum) mod aarch64 { use super::super::aarch64 as arch; use crate::checksum::dispatchers::Crc24Fn; /// OpenPGP PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)]. - pub const OPENPGP_PMULL: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_PMULL: [Crc24Fn; 5] = [ arch::crc24_openpgp_pmull_safe, arch::crc24_openpgp_pmull_2way_safe, arch::crc24_openpgp_pmull_3way_safe, @@ -75,16 +75,16 @@ pub mod aarch64 { ]; /// OpenPGP PMULL small-buffer kernel. - pub const OPENPGP_PMULL_SMALL_KERNEL: Crc24Fn = arch::crc24_openpgp_pmull_small_safe; + pub(in crate::checksum) const OPENPGP_PMULL_SMALL_KERNEL: Crc24Fn = arch::crc24_openpgp_pmull_small_safe; } #[cfg(target_arch = "powerpc64")] -pub mod power { +pub(in crate::checksum) mod power { use super::super::power as arch; use crate::checksum::dispatchers::Crc24Fn; /// OpenPGP VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)]. - pub const OPENPGP_VPMSUM: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_VPMSUM: [Crc24Fn; 5] = [ arch::crc24_openpgp_vpmsum_safe, arch::crc24_openpgp_vpmsum_2way_safe, arch::crc24_openpgp_vpmsum_4way_safe, @@ -94,12 +94,12 @@ pub mod power { } #[cfg(target_arch = "s390x")] -pub mod s390x { +pub(in crate::checksum) mod s390x { use super::super::s390x as arch; use crate::checksum::dispatchers::Crc24Fn; /// OpenPGP VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const OPENPGP_VGFM: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_VGFM: [Crc24Fn; 5] = [ arch::crc24_openpgp_vgfm_safe, arch::crc24_openpgp_vgfm_2way_safe, arch::crc24_openpgp_vgfm_4way_safe, @@ -109,31 +109,12 @@ pub mod s390x { } #[cfg(target_arch = "riscv64")] -#[allow(dead_code)] -pub mod riscv64 { +pub(in crate::checksum) mod riscv64 { use super::super::riscv64 as arch; use crate::checksum::dispatchers::Crc24Fn; - /// Zbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const ZBC_NAMES: &[&str] = &[ - "riscv64/zbc", - "riscv64/zbc-2way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", // dup for index consistency - "riscv64/zbc-4way", // dup for index consistency - ]; - - /// Zvbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const ZVBC_NAMES: &[&str] = &[ - "riscv64/zvbc", - "riscv64/zvbc-2way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", // dup for index consistency - "riscv64/zvbc-4way", // dup for index consistency - ]; - /// OpenPGP Zbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const OPENPGP_ZBC: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_ZBC: [Crc24Fn; 5] = [ arch::crc24_openpgp_zbc_safe, arch::crc24_openpgp_zbc_2way_safe, arch::crc24_openpgp_zbc_4way_safe, @@ -142,7 +123,7 @@ pub mod riscv64 { ]; /// OpenPGP Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const OPENPGP_ZVBC: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_ZVBC: [Crc24Fn; 5] = [ arch::crc24_openpgp_zvbc_safe, arch::crc24_openpgp_zvbc_2way_safe, arch::crc24_openpgp_zvbc_4way_safe, diff --git a/src/checksum/crc24/keys.rs b/src/checksum/crc24/keys.rs index 5587d653..0055f527 100644 --- a/src/checksum/crc24/keys.rs +++ b/src/checksum/crc24/keys.rs @@ -21,43 +21,75 @@ use crate::checksum::common::tables::CRC24_OPENPGP_POLY; /// CRC-24/OPENPGP polynomial in reflected CRC-32 form (LSB-first). /// /// This is `reverse_bits(poly32)` where `poly32 = CRC24_OPENPGP_POLY << 8`. -pub(crate) const CRC24_OPENPGP_POLY_REFLECTED: u32 = (CRC24_OPENPGP_POLY << 8).reverse_bits(); +pub(super) const CRC24_OPENPGP_POLY_REFLECTED: u32 = (CRC24_OPENPGP_POLY << 8).reverse_bits(); /// Key schedule for CRC-24/OPENPGP in the width32 folding strategy. #[rustfmt::skip] -pub(crate) const CRC24_OPENPGP_KEYS_REFLECTED: [u64; 23] = build_keys(CRC24_OPENPGP_POLY_REFLECTED); +pub(super) const CRC24_OPENPGP_KEYS_REFLECTED: [u64; 23] = build_keys(CRC24_OPENPGP_POLY_REFLECTED); /// Multi-stream folding constants for CRC-24/OPENPGP in the width32 strategy. /// /// These constants enable multi-way striping (2-way / 3-way / 4-way / 7-way / 8-way) /// for carryless-multiply kernels. #[derive(Clone, Copy, Debug)] -#[allow(dead_code)] // Field subsets vary by architecture (x86_64/aarch64/power/s390x/riscv64 stream widths). -pub(crate) struct Crc24StreamConstants { - pub fold_256b: (u64, u64), - pub fold_384b: (u64, u64), - pub fold_512b: (u64, u64), - pub fold_896b: (u64, u64), - pub fold_1024b: (u64, u64), - pub combine_4way: [(u64, u64); 3], - pub combine_7way: [(u64, u64); 6], - pub combine_8way: [(u64, u64); 7], +pub(super) struct Crc24StreamConstants { + pub(super) fold_256b: (u64, u64), + #[cfg(target_arch = "aarch64")] + pub(super) fold_384b: (u64, u64), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] + pub(super) fold_512b: (u64, u64), + #[cfg(target_arch = "x86_64")] + pub(super) fold_896b: (u64, u64), + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] + pub(super) fold_1024b: (u64, u64), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] + pub(super) combine_4way: [(u64, u64); 3], + #[cfg(target_arch = "x86_64")] + pub(super) combine_7way: [(u64, u64); 6], + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] + pub(super) combine_8way: [(u64, u64); 7], } impl Crc24StreamConstants { #[must_use] - pub const fn new(reflected_poly: u32) -> Self { + const fn new(reflected_poly: u32) -> Self { Self { fold_256b: fold16_coeff_for_bytes(reflected_poly, 256), + #[cfg(target_arch = "aarch64")] fold_384b: fold16_coeff_for_bytes(reflected_poly, 384), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] fold_512b: fold16_coeff_for_bytes(reflected_poly, 512), + #[cfg(target_arch = "x86_64")] fold_896b: fold16_coeff_for_bytes(reflected_poly, 896), + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] fold_1024b: fold16_coeff_for_bytes(reflected_poly, 1024), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] combine_4way: [ fold16_coeff_for_bytes(reflected_poly, 384), fold16_coeff_for_bytes(reflected_poly, 256), fold16_coeff_for_bytes(reflected_poly, 128), ], + #[cfg(target_arch = "x86_64")] combine_7way: [ fold16_coeff_for_bytes(reflected_poly, 768), fold16_coeff_for_bytes(reflected_poly, 640), @@ -66,6 +98,7 @@ impl Crc24StreamConstants { fold16_coeff_for_bytes(reflected_poly, 256), fold16_coeff_for_bytes(reflected_poly, 128), ], + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] combine_8way: [ fold16_coeff_for_bytes(reflected_poly, 896), fold16_coeff_for_bytes(reflected_poly, 768), @@ -79,7 +112,7 @@ impl Crc24StreamConstants { } } -pub(crate) const CRC24_OPENPGP_STREAM_REFLECTED: Crc24StreamConstants = +pub(super) const CRC24_OPENPGP_STREAM_REFLECTED: Crc24StreamConstants = Crc24StreamConstants::new(CRC24_OPENPGP_POLY_REFLECTED); // Constant Generation (compile-time) @@ -112,16 +145,16 @@ const fn reduce128(hi: u64, lo: u64, poly: u32) -> u32 { let poly_full: u128 = (1u128.strict_shl(32)) | (poly as u128); let mut val: u128 = (hi as u128).strict_shl(64) | (lo as u128); - let mut bit: i32 = 127; + let mut bit: u32 = 127; while bit >= 32 { - let b = bit as u32; - if ((val.strict_shr(b)) & 1) != 0 { - val ^= poly_full.strict_shl(b.strict_sub(32)); + if ((val.strict_shr(bit)) & 1) != 0 { + val ^= poly_full.strict_shl(bit.strict_sub(32)); } bit = bit.strict_sub(1); } - val as u32 + let [b0, b1, b2, b3, ..] = val.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) } /// Compute x^n mod (x^width + poly) in GF(2) where `poly` is the normal CRC polynomial diff --git a/src/checksum/crc24/mod.rs b/src/checksum/crc24/mod.rs index 6dc423bd..40c94eff 100644 --- a/src/checksum/crc24/mod.rs +++ b/src/checksum/crc24/mod.rs @@ -3,25 +3,15 @@ //! This module provides: //! - [`Crc24OpenPgp`] - CRC-24/OPENPGP (RFC 4880) //! -//! # Quick Start -//! -//! ```rust -//! use rscrypto::checksum::{Checksum, ChecksumCombine, Crc24OpenPgp}; -//! -//! let data = b"123456789"; -//! assert_eq!(Crc24OpenPgp::checksum(data), 0x21CF02); -//! -//! let (a, b) = data.split_at(4); -//! let combined = Crc24OpenPgp::combine( -//! Crc24OpenPgp::checksum(a), -//! Crc24OpenPgp::checksum(b), -//! b.len(), -//! ); -//! assert_eq!(combined, Crc24OpenPgp::checksum(data)); -//! ``` - pub(crate) mod config; pub(crate) mod kernels; +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" +))] pub(crate) mod keys; pub(crate) mod portable; #[cfg(any( @@ -33,7 +23,6 @@ pub(crate) mod portable; ))] mod reflected; -#[allow(unused_imports)] pub use config::{Crc24Config, Crc24Force}; #[cfg(any(test, feature = "std"))] @@ -42,10 +31,6 @@ use crate::checksum::common::{ combine::{Gf2Matrix24, combine_crc24, generate_shift8_matrix_24}, tables::{CRC24_OPENPGP_POLY, generate_crc24_tables_8}, }; -// Re-export traits for test modules (`use super::*`). -#[allow(unused_imports)] -pub(super) use crate::traits::{Checksum, ChecksumCombine}; - #[cfg(target_arch = "aarch64")] mod aarch64; #[cfg(target_arch = "powerpc64")] @@ -61,7 +46,7 @@ mod x86_64; mod kernel_tables { use super::*; - pub static OPENPGP_TABLES_8: [[u32; 256]; 8] = generate_crc24_tables_8(CRC24_OPENPGP_POLY); + pub(super) static OPENPGP_TABLES_8: [[u32; 256]; 8] = generate_crc24_tables_8(CRC24_OPENPGP_POLY); } // Reference Kernel Wrapper @@ -331,6 +316,7 @@ impl crate::traits::ChecksumCombine for Crc24OpenPgp { #[cfg(feature = "alloc")] impl Crc24OpenPgp { + /// Creates a buffering wrapper that coalesces short updates before CRC-24/OpenPGP dispatch. #[must_use] pub fn buffered() -> BufferedCrc24OpenPgp { BufferedCrc24OpenPgp::new() @@ -373,6 +359,7 @@ mod tests { extern crate std; use super::*; + use crate::traits::{Checksum, ChecksumCombine}; #[test] fn test_vectors_crc24_openpgp() { @@ -429,6 +416,7 @@ mod cross_check { use super::*; use crate::checksum::common::tests::{STREAMING_CHUNK_SIZES, TEST_LENGTHS, generate_test_data}; + use crate::traits::{Checksum, ChecksumCombine}; // CRC-24/OPENPGP Cross-Check Tests diff --git a/src/checksum/crc24/portable.rs b/src/checksum/crc24/portable.rs index 0afe0baf..b818e597 100644 --- a/src/checksum/crc24/portable.rs +++ b/src/checksum/crc24/portable.rs @@ -5,7 +5,7 @@ use crate::checksum::common::portable; /// CRC-24/OPENPGP slice-by-8 computation. #[inline] -pub fn crc24_openpgp_slice8(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc24_openpgp_slice8(crc: u32, data: &[u8]) -> u32 { portable::slice8_24(crc, data, &kernel_tables::OPENPGP_TABLES_8) } @@ -15,8 +15,7 @@ pub fn crc24_openpgp_slice8(crc: u32, data: &[u8]) -> u32 { /// /// Uses one 256-entry table rather than the slice-by-8 table set. #[inline(always)] -#[allow(clippy::indexing_slicing)] // index is 0..=255 by byte cast, table is [u32; 256] -pub fn crc24_openpgp_bytewise(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc24_openpgp_bytewise(crc: u32, data: &[u8]) -> u32 { const MASK24: u32 = 0x00FF_FFFF; let mut state = (crc & MASK24) << 8; for &byte in data { diff --git a/src/checksum/crc24/power.rs b/src/checksum/crc24/power.rs index 0e00b74f..17447a4a 100644 --- a/src/checksum/crc24/power.rs +++ b/src/checksum/crc24/power.rs @@ -9,10 +9,7 @@ //! Uses `unsafe` for Power SIMD + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher // SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, @@ -20,6 +17,8 @@ use core::{ simd::i64x2, }; +use crate::checksum::common::low_u32; + use super::{ keys::{CRC24_OPENPGP_KEYS_REFLECTED, CRC24_OPENPGP_STREAM_REFLECTED}, reflected::{crc24_reflected_update_bitrev_bytes, from_reflected_state, to_reflected_state}, @@ -87,11 +86,13 @@ impl Simd { } /// Normalize a loaded vector to little-endian lane encoding. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn to_le(self) -> Self { - // SAFETY: All intrinsics and asm require altivec+vsx+power8-vector, ensured by this function's - // #[target_feature] attribute. #[cfg(target_endian = "little")] { self @@ -117,6 +118,9 @@ impl Simd { Self::new(bitrev_bytes_u64(self.high_64()), bitrev_bytes_u64(self.low_64())) } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -140,6 +144,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -157,6 +164,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -170,6 +180,9 @@ impl Simd { unsafe { Self(Self::vpmsumd(self.0, coeff.swap_lanes().0)) } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -183,6 +196,9 @@ impl Simd { unsafe { data_to_xor ^ self.fold_16(coeff) } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -208,6 +224,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -221,11 +240,14 @@ impl Simd { unsafe { let t1 = Self::mul64(self.low_64(), mu); let l = Self::mul64(t1.low_64(), poly); - (self ^ l).high_64() as u32 + low_u32((self ^ l).high_64()) } } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -251,6 +273,9 @@ unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u3 } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -295,6 +320,9 @@ unsafe fn update_simd_width32_reflected_bitrev( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -317,6 +345,9 @@ unsafe fn fold_block_128_reflected_bitrev(x: &mut [Simd; 8], chunk: &[Simd; 8], } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -340,6 +371,9 @@ unsafe fn normalize_block_le_bitrev(mut block: [Simd; 8]) -> [Simd; 8] { } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -400,6 +434,9 @@ unsafe fn update_simd_width32_reflected_bitrev_2way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -426,7 +463,7 @@ unsafe fn update_simd_width32_reflected_bitrev_4way( return update_simd_width32_reflected_bitrev(state, first, rest, keys); } - let aligned = (blocks.len() / 4) * 4; + let aligned = blocks.len() & !3_usize; let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); let coeff_128 = Simd::new(keys[4], keys[3]); @@ -487,6 +524,9 @@ unsafe fn update_simd_width32_reflected_bitrev_4way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -513,7 +553,7 @@ unsafe fn update_simd_width32_reflected_bitrev_8way( return update_simd_width32_reflected_bitrev(state, first, rest, keys); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len() & !7_usize; let coeff_1024 = Simd::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd::new(keys[4], keys[3]); @@ -622,6 +662,9 @@ unsafe fn update_simd_width32_reflected_bitrev_8way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -645,6 +688,9 @@ unsafe fn crc24_width32_vpmsum_bitrev(mut state: u32, data: &[u8], keys: &[u64; } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -668,6 +714,9 @@ unsafe fn crc24_width32_vpmsum_bitrev_2way(mut state: u32, data: &[u8], keys: &[ } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -697,6 +746,9 @@ unsafe fn crc24_width32_vpmsum_bitrev_4way(mut state: u32, data: &[u8], keys: &[ } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -730,11 +782,9 @@ unsafe fn crc24_width32_vpmsum_bitrev_8way(mut state: u32, data: &[u8], keys: &[ /// CRC-24/OPENPGP VPMSUMD kernel. /// -/// # Safety -/// -/// Dispatcher verifies VPMSUMD before selecting this kernel. +/// Runtime dispatch selects this kernel only when VPMSUMD is available. #[inline] -pub fn crc24_openpgp_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. state = unsafe { crc24_width32_vpmsum_bitrev(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -742,7 +792,7 @@ pub fn crc24_openpgp_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. state = unsafe { crc24_width32_vpmsum_bitrev_2way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -750,7 +800,7 @@ pub fn crc24_openpgp_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. state = unsafe { crc24_width32_vpmsum_bitrev_4way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -758,7 +808,7 @@ pub fn crc24_openpgp_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. state = unsafe { crc24_width32_vpmsum_bitrev_8way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; diff --git a/src/checksum/crc24/reflected.rs b/src/checksum/crc24/reflected.rs index ae059085..8540e29f 100644 --- a/src/checksum/crc24/reflected.rs +++ b/src/checksum/crc24/reflected.rs @@ -9,7 +9,6 @@ // SAFETY: All array indexing in this module uses bounded indices: // - Table generation iterates i: 0..256 into a [u32; 256] // - Table lookups use `& 0xFF` which produces indices 0..255 -#![allow(clippy::indexing_slicing)] use super::keys::CRC24_OPENPGP_POLY_REFLECTED; @@ -42,10 +41,13 @@ const fn crc32_reflected_table_entry(poly: u32, index: u8) -> u32 { #[must_use] const fn generate_crc32_reflected_table(poly: u32) -> [u32; 256] { let mut table = [0u32; 256]; - let mut i: u16 = 0; - while i < 256 { - table[i as usize] = crc32_reflected_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + table[index as usize] = crc32_reflected_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } table } diff --git a/src/checksum/crc24/riscv64.rs b/src/checksum/crc24/riscv64.rs index 5e1bd1a5..fcaec847 100644 --- a/src/checksum/crc24/riscv64.rs +++ b/src/checksum/crc24/riscv64.rs @@ -9,17 +9,14 @@ //! Uses `unsafe` for RISC-V inline assembly. Callers must ensure the required //! CPU features are available before executing the accelerated path (the //! dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, - mem::MaybeUninit, ops::{BitXor, BitXorAssign}, }; +use crate::checksum::common::low_u32; + use super::{ keys::{CRC24_OPENPGP_KEYS_REFLECTED, CRC24_OPENPGP_STREAM_REFLECTED}, reflected::{crc24_reflected_update_bitrev_bytes, from_reflected_state, to_reflected_state}, @@ -72,11 +69,11 @@ impl Simd { // Zbc carryless multiply primitives #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn clmul_lo(a: u64, b: u64) -> u64 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + fn clmul_lo(a: u64, b: u64) -> u64 { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -93,11 +90,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn clmul_hi(a: u64, b: u64) -> u64 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + fn clmul_hi(a: u64, b: u64) -> u64 { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -114,18 +111,14 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - Self { - hi: Self::clmul_hi(a, b), - lo: Self::clmul_lo(a, b), - } + fn mul64(a: u64, b: u64) -> Self { + Self { + hi: Self::clmul_hi(a, b), + lo: Self::clmul_lo(a, b), } } } @@ -143,34 +136,23 @@ const fn bitrev_bytes_u64(mut x: u64) -> u64 { #[inline(always)] fn load_block_bitrev(block: &Block) -> [Simd; 8] { - let mut out = MaybeUninit::<[Simd; 8]>::uninit(); - let base = out.as_mut_ptr().cast::(); - - let mut i = 0usize; - while i < 8 { - let lo = bitrev_bytes_u64(u64::from_le(block[i * 2])); - let hi = bitrev_bytes_u64(u64::from_le(block[i * 2 + 1])); - // SAFETY: `base` points to a `[Simd; 8]` buffer and `i` is in-bounds. - unsafe { - base.add(i).write(Simd::new(hi, lo)); - } - i = i.strict_add(1); + let mut out = [Simd::new(0, 0); 8]; + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new( + bitrev_bytes_u64(u64::from_le(high)), + bitrev_bytes_u64(u64::from_le(low)), + ); } - - // SAFETY: all 8 elements are initialized above. - unsafe { out.assume_init() } + out } #[inline] fn load_block_split_bitrev(block: &Block) -> ([u64; 8], [u64; 8]) { let mut hi = [0u64; 8]; let mut lo = [0u64; 8]; - - let mut i = 0usize; - while i < 8 { - lo[i] = bitrev_bytes_u64(u64::from_le(block[i * 2])); - hi[i] = bitrev_bytes_u64(u64::from_le(block[i * 2 + 1])); - i = i.strict_add(1); + for ((hi_lane, lo_lane), &[low, high]) in hi.iter_mut().zip(&mut lo).zip(block.as_chunks::<2>().0) { + *hi_lane = bitrev_bytes_u64(u64::from_le(high)); + *lo_lane = bitrev_bytes_u64(u64::from_le(low)); } (hi, lo) @@ -179,277 +161,249 @@ fn load_block_split_bitrev(block: &Block) -> ([u64; 8], [u64; 8]) { // ZBC (scalar carryless multiply) backend #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_16_zbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - Simd::mul64(x.low_64(), coeff_high) ^ Simd::mul64(x.high_64(), coeff_low) - } +fn fold_16_zbc(x: Simd, coeff: (u64, u64)) -> Simd { + let (coeff_high, coeff_low) = coeff; + Simd::mul64(x.low_64(), coeff_high) ^ Simd::mul64(x.high_64(), coeff_low) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_16_reflected_zbc(x: Simd, coeff: (u64, u64), data_to_xor: Simd) -> Simd { - // SAFETY: Caller guarantees: - // 1. ZBC + ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { data_to_xor ^ fold_16_zbc(x, coeff) } +fn fold_16_reflected_zbc(x: Simd, coeff: (u64, u64), data_to_xor: Simd) -> Simd { + data_to_xor ^ fold_16_zbc(x, coeff) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_width32_reflected_zbc(x: Simd, high: u64, low: u64) -> Simd { - // SAFETY: Caller guarantees: - // 1. ZBC + ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let clmul = Simd::mul64(x.low_64(), low); - let shifted = Simd::new(0, x.high_64()); - let mut state = clmul ^ shifted; +fn fold_width32_reflected_zbc(x: Simd, high: u64, low: u64) -> Simd { + let clmul = Simd::mul64(x.low_64(), low); + let shifted = Simd::new(0, x.high_64()); + let mut state = clmul ^ shifted; - let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = Simd::mul64(shifted_high, high); - state = clmul ^ masked; + let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = Simd::mul64(shifted_high, high); + state = clmul ^ masked; - state - } + state } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn barrett_width32_reflected_zbc(x: Simd, poly: u64, mu: u64) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = Simd::mul64(x.low_64(), mu); - let l = Simd::mul64(t1.low_64(), poly); - (x ^ l).high_64() as u32 - } +fn barrett_width32_reflected_zbc(x: Simd, poly: u64, mu: u64) -> u32 { + let t1 = Simd::mul64(x.low_64(), mu); + let l = Simd::mul64(t1.low_64(), poly); + low_u32((x ^ l).high_64()) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn finalize_lanes_width32_reflected_zbc(x: [Simd; 8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut res = x[7]; - res = fold_16_reflected_zbc(x[0], (keys[10], keys[9]), res); - res = fold_16_reflected_zbc(x[1], (keys[12], keys[11]), res); - res = fold_16_reflected_zbc(x[2], (keys[14], keys[13]), res); - res = fold_16_reflected_zbc(x[3], (keys[16], keys[15]), res); - res = fold_16_reflected_zbc(x[4], (keys[18], keys[17]), res); - res = fold_16_reflected_zbc(x[5], (keys[20], keys[19]), res); - res = fold_16_reflected_zbc(x[6], (keys[2], keys[1]), res); - - barrett_width32_reflected_zbc(fold_width32_reflected_zbc(res, keys[6], keys[5]), keys[8], keys[7]) - } +fn finalize_lanes_width32_reflected_zbc(x: [Simd; 8], keys: &[u64; 23]) -> u32 { + let mut res = x[7]; + res = fold_16_reflected_zbc(x[0], (keys[10], keys[9]), res); + res = fold_16_reflected_zbc(x[1], (keys[12], keys[11]), res); + res = fold_16_reflected_zbc(x[2], (keys[14], keys[13]), res); + res = fold_16_reflected_zbc(x[3], (keys[16], keys[15]), res); + res = fold_16_reflected_zbc(x[4], (keys[18], keys[17]), res); + res = fold_16_reflected_zbc(x[5], (keys[20], keys[19]), res); + res = fold_16_reflected_zbc(x[6], (keys[2], keys[1]), res); + + barrett_width32_reflected_zbc(fold_width32_reflected_zbc(res, keys[6], keys[5]), keys[8], keys[7]) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut x = load_block_bitrev(first); - x[0] ^= Simd::new(0, state as u64); - - let coeff_128b = (keys[4], keys[3]); - for block in rest { - let chunk = load_block_bitrev(block); - x[0] = fold_16_reflected_zbc(x[0], coeff_128b, chunk[0]); - x[1] = fold_16_reflected_zbc(x[1], coeff_128b, chunk[1]); - x[2] = fold_16_reflected_zbc(x[2], coeff_128b, chunk[2]); - x[3] = fold_16_reflected_zbc(x[3], coeff_128b, chunk[3]); - x[4] = fold_16_reflected_zbc(x[4], coeff_128b, chunk[4]); - x[5] = fold_16_reflected_zbc(x[5], coeff_128b, chunk[5]); - x[6] = fold_16_reflected_zbc(x[6], coeff_128b, chunk[6]); - x[7] = fold_16_reflected_zbc(x[7], coeff_128b, chunk[7]); - } +fn update_simd_zbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { + let mut x = load_block_bitrev(first); + x[0] ^= Simd::new(0, state as u64); - finalize_lanes_width32_reflected_zbc(x, keys) + let coeff_128b = (keys[4], keys[3]); + for block in rest { + let chunk = load_block_bitrev(block); + x[0] = fold_16_reflected_zbc(x[0], coeff_128b, chunk[0]); + x[1] = fold_16_reflected_zbc(x[1], coeff_128b, chunk[1]); + x[2] = fold_16_reflected_zbc(x[2], coeff_128b, chunk[2]); + x[3] = fold_16_reflected_zbc(x[3], coeff_128b, chunk[3]); + x[4] = fold_16_reflected_zbc(x[4], coeff_128b, chunk[4]); + x[5] = fold_16_reflected_zbc(x[5], coeff_128b, chunk[5]); + x[6] = fold_16_reflected_zbc(x[6], coeff_128b, chunk[6]); + x[7] = fold_16_reflected_zbc(x[7], coeff_128b, chunk[7]); } + + finalize_lanes_width32_reflected_zbc(x, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_block_128_reflected_zbc(x: &mut [Simd; 8], block: &Block, coeff: (u64, u64)) { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let chunk = load_block_bitrev(block); - x[0] = fold_16_reflected_zbc(x[0], coeff, chunk[0]); - x[1] = fold_16_reflected_zbc(x[1], coeff, chunk[1]); - x[2] = fold_16_reflected_zbc(x[2], coeff, chunk[2]); - x[3] = fold_16_reflected_zbc(x[3], coeff, chunk[3]); - x[4] = fold_16_reflected_zbc(x[4], coeff, chunk[4]); - x[5] = fold_16_reflected_zbc(x[5], coeff, chunk[5]); - x[6] = fold_16_reflected_zbc(x[6], coeff, chunk[6]); - x[7] = fold_16_reflected_zbc(x[7], coeff, chunk[7]); - } +fn fold_block_128_reflected_zbc(x: &mut [Simd; 8], block: &Block, coeff: (u64, u64)) { + let chunk = load_block_bitrev(block); + x[0] = fold_16_reflected_zbc(x[0], coeff, chunk[0]); + x[1] = fold_16_reflected_zbc(x[1], coeff, chunk[1]); + x[2] = fold_16_reflected_zbc(x[2], coeff, chunk[2]); + x[3] = fold_16_reflected_zbc(x[3], coeff, chunk[3]); + x[4] = fold_16_reflected_zbc(x[4], coeff, chunk[4]); + x[5] = fold_16_reflected_zbc(x[5], coeff, chunk[5]); + x[6] = fold_16_reflected_zbc(x[6], coeff, chunk[6]); + x[7] = fold_16_reflected_zbc(x[7], coeff, chunk[7]); } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); +fn update_simd_zbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { + debug_assert!(!blocks.is_empty()); - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zbc(state, first, rest, keys); - } + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zbc(state, first, rest, keys); + } - let even = blocks.len() & !1usize; - let coeff_256 = fold_256b; - let coeff_128 = (keys[4], keys[3]); + let even = blocks.len() & !1usize; + let coeff_256 = fold_256b; + let coeff_128 = (keys[4], keys[3]); - let mut s0 = load_block_bitrev(&blocks[0]); - let mut s1 = load_block_bitrev(&blocks[1]); + let mut s0 = load_block_bitrev(&blocks[0]); + let mut s1 = load_block_bitrev(&blocks[1]); - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, state as u64); - let mut i: usize = 2; - while i < even { - fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_256); - fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_256); - i = i.strict_add(2); - } + let mut i: usize = 2; + while i < even { + fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_256); + fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_256); + i = i.strict_add(2); + } - // Merge: A·s0 ⊕ s1 (A = shift by 128B). - s1[0] = fold_16_reflected_zbc(s0[0], coeff_128, s1[0]); - s1[1] = fold_16_reflected_zbc(s0[1], coeff_128, s1[1]); - s1[2] = fold_16_reflected_zbc(s0[2], coeff_128, s1[2]); - s1[3] = fold_16_reflected_zbc(s0[3], coeff_128, s1[3]); - s1[4] = fold_16_reflected_zbc(s0[4], coeff_128, s1[4]); - s1[5] = fold_16_reflected_zbc(s0[5], coeff_128, s1[5]); - s1[6] = fold_16_reflected_zbc(s0[6], coeff_128, s1[6]); - s1[7] = fold_16_reflected_zbc(s0[7], coeff_128, s1[7]); - - if even != blocks.len() { - fold_block_128_reflected_zbc(&mut s1, &blocks[even], coeff_128); - } + // Merge: A·s0 ⊕ s1 (A = shift by 128B). + s1[0] = fold_16_reflected_zbc(s0[0], coeff_128, s1[0]); + s1[1] = fold_16_reflected_zbc(s0[1], coeff_128, s1[1]); + s1[2] = fold_16_reflected_zbc(s0[2], coeff_128, s1[2]); + s1[3] = fold_16_reflected_zbc(s0[3], coeff_128, s1[3]); + s1[4] = fold_16_reflected_zbc(s0[4], coeff_128, s1[4]); + s1[5] = fold_16_reflected_zbc(s0[5], coeff_128, s1[5]); + s1[6] = fold_16_reflected_zbc(s0[6], coeff_128, s1[6]); + s1[7] = fold_16_reflected_zbc(s0[7], coeff_128, s1[7]); - finalize_lanes_width32_reflected_zbc(s1, keys) + if even != blocks.len() { + fold_block_128_reflected_zbc(&mut s1, &blocks[even], coeff_128); } + + finalize_lanes_width32_reflected_zbc(s1, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc_4way( +fn update_simd_zbc_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); + debug_assert!(!blocks.is_empty()); - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zbc(state, first, rest, keys); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zbc(state, first, rest, keys); + } - let aligned = blocks.len().strict_div(4).strict_mul(4); + let aligned = blocks.len().strict_div(4).strict_mul(4); - let coeff_512 = fold_512b; - let coeff_128 = (keys[4], keys[3]); - let c384 = combine[0]; - let c256 = combine[1]; - let c128 = combine[2]; + let coeff_512 = fold_512b; + let coeff_128 = (keys[4], keys[3]); + let c384 = combine[0]; + let c256 = combine[1]; + let c128 = combine[2]; - let mut s0 = load_block_bitrev(&blocks[0]); - let mut s1 = load_block_bitrev(&blocks[1]); - let mut s2 = load_block_bitrev(&blocks[2]); - let mut s3 = load_block_bitrev(&blocks[3]); + let mut s0 = load_block_bitrev(&blocks[0]); + let mut s1 = load_block_bitrev(&blocks[1]); + let mut s2 = load_block_bitrev(&blocks[2]); + let mut s3 = load_block_bitrev(&blocks[3]); - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, state as u64); - let mut i: usize = 4; - while i < aligned { - fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_512); - fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_512); - fold_block_128_reflected_zbc(&mut s2, &blocks[i.strict_add(2)], coeff_512); - fold_block_128_reflected_zbc(&mut s3, &blocks[i.strict_add(3)], coeff_512); - i = i.strict_add(4); - } + let mut i: usize = 4; + while i < aligned { + fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_512); + fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128_reflected_zbc(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128_reflected_zbc(&mut s3, &blocks[i.strict_add(3)], coeff_512); + i = i.strict_add(4); + } - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - s3[0] = fold_16_reflected_zbc(s2[0], c128, s3[0]); - s3[1] = fold_16_reflected_zbc(s2[1], c128, s3[1]); - s3[2] = fold_16_reflected_zbc(s2[2], c128, s3[2]); - s3[3] = fold_16_reflected_zbc(s2[3], c128, s3[3]); - s3[4] = fold_16_reflected_zbc(s2[4], c128, s3[4]); - s3[5] = fold_16_reflected_zbc(s2[5], c128, s3[5]); - s3[6] = fold_16_reflected_zbc(s2[6], c128, s3[6]); - s3[7] = fold_16_reflected_zbc(s2[7], c128, s3[7]); - - s3[0] = fold_16_reflected_zbc(s1[0], c256, s3[0]); - s3[1] = fold_16_reflected_zbc(s1[1], c256, s3[1]); - s3[2] = fold_16_reflected_zbc(s1[2], c256, s3[2]); - s3[3] = fold_16_reflected_zbc(s1[3], c256, s3[3]); - s3[4] = fold_16_reflected_zbc(s1[4], c256, s3[4]); - s3[5] = fold_16_reflected_zbc(s1[5], c256, s3[5]); - s3[6] = fold_16_reflected_zbc(s1[6], c256, s3[6]); - s3[7] = fold_16_reflected_zbc(s1[7], c256, s3[7]); - - s3[0] = fold_16_reflected_zbc(s0[0], c384, s3[0]); - s3[1] = fold_16_reflected_zbc(s0[1], c384, s3[1]); - s3[2] = fold_16_reflected_zbc(s0[2], c384, s3[2]); - s3[3] = fold_16_reflected_zbc(s0[3], c384, s3[3]); - s3[4] = fold_16_reflected_zbc(s0[4], c384, s3[4]); - s3[5] = fold_16_reflected_zbc(s0[5], c384, s3[5]); - s3[6] = fold_16_reflected_zbc(s0[6], c384, s3[6]); - s3[7] = fold_16_reflected_zbc(s0[7], c384, s3[7]); - - for block in &blocks[aligned..] { - fold_block_128_reflected_zbc(&mut s3, block, coeff_128); - } + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + s3[0] = fold_16_reflected_zbc(s2[0], c128, s3[0]); + s3[1] = fold_16_reflected_zbc(s2[1], c128, s3[1]); + s3[2] = fold_16_reflected_zbc(s2[2], c128, s3[2]); + s3[3] = fold_16_reflected_zbc(s2[3], c128, s3[3]); + s3[4] = fold_16_reflected_zbc(s2[4], c128, s3[4]); + s3[5] = fold_16_reflected_zbc(s2[5], c128, s3[5]); + s3[6] = fold_16_reflected_zbc(s2[6], c128, s3[6]); + s3[7] = fold_16_reflected_zbc(s2[7], c128, s3[7]); + + s3[0] = fold_16_reflected_zbc(s1[0], c256, s3[0]); + s3[1] = fold_16_reflected_zbc(s1[1], c256, s3[1]); + s3[2] = fold_16_reflected_zbc(s1[2], c256, s3[2]); + s3[3] = fold_16_reflected_zbc(s1[3], c256, s3[3]); + s3[4] = fold_16_reflected_zbc(s1[4], c256, s3[4]); + s3[5] = fold_16_reflected_zbc(s1[5], c256, s3[5]); + s3[6] = fold_16_reflected_zbc(s1[6], c256, s3[6]); + s3[7] = fold_16_reflected_zbc(s1[7], c256, s3[7]); + + s3[0] = fold_16_reflected_zbc(s0[0], c384, s3[0]); + s3[1] = fold_16_reflected_zbc(s0[1], c384, s3[1]); + s3[2] = fold_16_reflected_zbc(s0[2], c384, s3[2]); + s3[3] = fold_16_reflected_zbc(s0[3], c384, s3[3]); + s3[4] = fold_16_reflected_zbc(s0[4], c384, s3[4]); + s3[5] = fold_16_reflected_zbc(s0[5], c384, s3[5]); + s3[6] = fold_16_reflected_zbc(s0[6], c384, s3[6]); + s3[7] = fold_16_reflected_zbc(s0[7], c384, s3[7]); - finalize_lanes_width32_reflected_zbc(s3, keys) + for block in &blocks[aligned..] { + fold_block_128_reflected_zbc(&mut s3, block, coeff_128); } + + finalize_lanes_width32_reflected_zbc(s3, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc24_width32_zbc(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc24_width32_zbc(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -463,13 +417,13 @@ unsafe fn crc24_width32_zbc(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u3 } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc24_width32_zbc_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc24_width32_zbc_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -483,13 +437,13 @@ unsafe fn crc24_width32_zbc_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc24_width32_zbc_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc24_width32_zbc_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -517,10 +471,7 @@ unsafe fn crc24_width32_zbc_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) /// Requires RISC-V `v` + `zvbc`. #[inline] #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. +fn mul64_zvbc(a: u64, b: u64) -> Simd { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -547,81 +498,68 @@ unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - mul64_zvbc(x.low_64(), coeff_high) ^ mul64_zvbc(x.high_64(), coeff_low) - } +fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { + let (coeff_high, coeff_low) = coeff; + mul64_zvbc(x.low_64(), coeff_high) ^ mul64_zvbc(x.high_64(), coeff_low) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_width32_reflected_zvbc(x: Simd, high: u64, low: u64) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC + V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let clmul = mul64_zvbc(x.low_64(), low); - let shifted = Simd::new(0, x.high_64()); - let mut state = clmul ^ shifted; +fn fold_width32_reflected_zvbc(x: Simd, high: u64, low: u64) -> Simd { + let clmul = mul64_zvbc(x.low_64(), low); + let shifted = Simd::new(0, x.high_64()); + let mut state = clmul ^ shifted; - let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = mul64_zvbc(shifted_high, high); - state = clmul ^ masked; + let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = mul64_zvbc(shifted_high, high); + state = clmul ^ masked; - state - } + state } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn barrett_width32_reflected_zvbc(x: Simd, poly: u64, mu: u64) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = mul64_zvbc(x.low_64(), mu); - let l = mul64_zvbc(t1.low_64(), poly); - (x ^ l).high_64() as u32 - } +fn barrett_width32_reflected_zvbc(x: Simd, poly: u64, mu: u64) -> u32 { + let t1 = mul64_zvbc(x.low_64(), mu); + let l = mul64_zvbc(t1.low_64(), poly); + low_u32((x ^ l).high_64()) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut acc = Simd::new(hi[7], lo[7]); - acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), (keys[10], keys[9])); - acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), (keys[12], keys[11])); - acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), (keys[14], keys[13])); - acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), (keys[16], keys[15])); - acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), (keys[18], keys[17])); - acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), (keys[20], keys[19])); - acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), (keys[2], keys[1])); - - barrett_width32_reflected_zvbc(fold_width32_reflected_zvbc(acc, keys[6], keys[5]), keys[8], keys[7]) - } +fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], keys: &[u64; 23]) -> u32 { + let mut acc = Simd::new(hi[7], lo[7]); + acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), (keys[10], keys[9])); + acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), (keys[12], keys[11])); + acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), (keys[14], keys[13])); + acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), (keys[16], keys[15])); + acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), (keys[18], keys[17])); + acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), (keys[20], keys[19])); + acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), (keys[2], keys[1])); + + barrett_width32_reflected_zvbc(fold_width32_reflected_zvbc(acc, keys[6], keys[5]), keys[8], keys[7]) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_block_128_zvbc( +fn fold_block_128_zvbc( x_hi: &mut [u64; 8], x_lo: &mut [u64; 8], chunk_hi: &[u64; 8], @@ -629,15 +567,12 @@ unsafe fn fold_block_128_zvbc( coeff_low: u64, coeff_high: u64, ) { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { let mut offset = 0usize; while offset < 8 { - let remaining = 8 - offset; + let remaining = 8usize.strict_sub(offset); let vl: usize; asm!( "vsetvli {vl}, {avl}, e64, m1, ta, ma", @@ -677,174 +612,162 @@ unsafe fn fold_block_128_zvbc( } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (mut x_hi, mut x_lo) = load_block_split_bitrev(first); - x_lo[0] ^= state as u64; - - let coeff_low = keys[3]; - let coeff_high = keys[4]; +fn update_simd_zvbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { + let (mut x_hi, mut x_lo) = load_block_split_bitrev(first); + x_lo[0] ^= state as u64; - for block in rest { - let (chunk_hi, chunk_lo) = load_block_split_bitrev(block); - fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); - } + let coeff_low = keys[3]; + let coeff_high = keys[4]; - fold_tail_zvbc(x_hi, x_lo, keys) + for block in rest { + let (chunk_hi, chunk_lo) = load_block_split_bitrev(block); + fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); } + + fold_tail_zvbc(x_hi, x_lo, keys) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); +fn update_simd_zvbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { + debug_assert!(!blocks.is_empty()); - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, keys); - } + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zvbc(state, first, rest, keys); + } - let even = blocks.len() & !1usize; + let even = blocks.len() & !1usize; - let coeff_256_low = fold_256b.1; - let coeff_256_high = fold_256b.0; - let coeff_128_low = keys[3]; - let coeff_128_high = keys[4]; + let coeff_256_low = fold_256b.1; + let coeff_256_high = fold_256b.0; + let coeff_128_low = keys[3]; + let coeff_128_high = keys[4]; - let (mut s0_hi, mut s0_lo) = load_block_split_bitrev(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split_bitrev(&blocks[1]); + let (mut s0_hi, mut s0_lo) = load_block_split_bitrev(&blocks[0]); + let (mut s1_hi, mut s1_lo) = load_block_split_bitrev(&blocks[1]); - // Inject CRC into stream 0. - s0_lo[0] ^= state as u64; + // Inject CRC into stream 0. + s0_lo[0] ^= state as u64; - let mut i: usize = 2; - while i < even { - let (b0_hi, b0_lo) = load_block_split_bitrev(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split_bitrev(&blocks[i.strict_add(1)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); - i = i.strict_add(2); - } + let mut i: usize = 2; + while i < even { + let (b0_hi, b0_lo) = load_block_split_bitrev(&blocks[i]); + let (b1_hi, b1_lo) = load_block_split_bitrev(&blocks[i.strict_add(1)]); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); + i = i.strict_add(2); + } - // Merge: A·s0 ⊕ s1 (A = shift by 128B). - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, coeff_128_low, coeff_128_high); - - if even != blocks.len() { - let (tail_hi, tail_lo) = load_block_split_bitrev(&blocks[even]); - fold_block_128_zvbc( - &mut s0_hi, - &mut s0_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } + // Merge: A·s0 ⊕ s1 (A = shift by 128B). + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, coeff_128_low, coeff_128_high); - fold_tail_zvbc(s0_hi, s0_lo, keys) + if even != blocks.len() { + let (tail_hi, tail_lo) = load_block_split_bitrev(&blocks[even]); + fold_block_128_zvbc( + &mut s0_hi, + &mut s0_lo, + &tail_hi, + &tail_lo, + coeff_128_low, + coeff_128_high, + ); } + + fold_tail_zvbc(s0_hi, s0_lo, keys) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_4way( +fn update_simd_zvbc_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, keys); - } - - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512_low = fold_512b.1; - let coeff_512_high = fold_512b.0; - let coeff_128_low = keys[3]; - let coeff_128_high = keys[4]; - - let c384_low = combine[0].1; - let c384_high = combine[0].0; - let c256_low = combine[1].1; - let c256_high = combine[1].0; - let c128_low = combine[2].1; - let c128_high = combine[2].0; - - let (mut s0_hi, mut s0_lo) = load_block_split_bitrev(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split_bitrev(&blocks[1]); - let (mut s2_hi, mut s2_lo) = load_block_split_bitrev(&blocks[2]); - let (mut s3_hi, mut s3_lo) = load_block_split_bitrev(&blocks[3]); - - // Inject CRC into stream 0. - s0_lo[0] ^= state as u64; - - let mut i: usize = 4; - while i < aligned { - let (b0_hi, b0_lo) = load_block_split_bitrev(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split_bitrev(&blocks[i.strict_add(1)]); - let (b2_hi, b2_lo) = load_block_split_bitrev(&blocks[i.strict_add(2)]); - let (b3_hi, b3_lo) = load_block_split_bitrev(&blocks[i.strict_add(3)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); - i = i.strict_add(4); - } - - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &s3_hi, &s3_lo, c128_low, c128_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &s2_hi, &s2_lo, c256_low, c256_high); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, c384_low, c384_high); - - for block in &blocks[aligned..] { - let (tail_hi, tail_lo) = load_block_split_bitrev(block); - fold_block_128_zvbc( - &mut s0_hi, - &mut s0_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } + debug_assert!(!blocks.is_empty()); - fold_tail_zvbc(s0_hi, s0_lo, keys) + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zvbc(state, first, rest, keys); + } + + let aligned = blocks.len().strict_div(4).strict_mul(4); + + let coeff_512_low = fold_512b.1; + let coeff_512_high = fold_512b.0; + let coeff_128_low = keys[3]; + let coeff_128_high = keys[4]; + + let c384_low = combine[0].1; + let c384_high = combine[0].0; + let c256_low = combine[1].1; + let c256_high = combine[1].0; + let c128_low = combine[2].1; + let c128_high = combine[2].0; + + let (mut s0_hi, mut s0_lo) = load_block_split_bitrev(&blocks[0]); + let (mut s1_hi, mut s1_lo) = load_block_split_bitrev(&blocks[1]); + let (mut s2_hi, mut s2_lo) = load_block_split_bitrev(&blocks[2]); + let (mut s3_hi, mut s3_lo) = load_block_split_bitrev(&blocks[3]); + + // Inject CRC into stream 0. + s0_lo[0] ^= state as u64; + + let mut i: usize = 4; + while i < aligned { + let (b0_hi, b0_lo) = load_block_split_bitrev(&blocks[i]); + let (b1_hi, b1_lo) = load_block_split_bitrev(&blocks[i.strict_add(1)]); + let (b2_hi, b2_lo) = load_block_split_bitrev(&blocks[i.strict_add(2)]); + let (b3_hi, b3_lo) = load_block_split_bitrev(&blocks[i.strict_add(3)]); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); + i = i.strict_add(4); + } + + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &s3_hi, &s3_lo, c128_low, c128_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &s2_hi, &s2_lo, c256_low, c256_high); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, c384_low, c384_high); + + for block in &blocks[aligned..] { + let (tail_hi, tail_lo) = load_block_split_bitrev(block); + fold_block_128_zvbc( + &mut s0_hi, + &mut s0_lo, + &tail_hi, + &tail_lo, + coeff_128_low, + coeff_128_high, + ); } + + fold_tail_zvbc(s0_hi, s0_lo, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc24_width32_zvbc(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc24_width32_zvbc(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -858,13 +781,13 @@ unsafe fn crc24_width32_zvbc(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc24_width32_zvbc_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc24_width32_zvbc_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -878,13 +801,13 @@ unsafe fn crc24_width32_zvbc_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc24_width32_zvbc_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc24_width32_zvbc_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -906,12 +829,8 @@ unsafe fn crc24_width32_zvbc_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) // Public Safe Kernels /// CRC-24/OPENPGP Zbc kernel. -/// -/// # Safety -/// -/// Dispatcher verifies Zbc before selecting this kernel. #[inline] -pub fn crc24_openpgp_zbc_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_zbc_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies Zbc before selecting this kernel. state = unsafe { crc24_width32_zbc(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -919,7 +838,7 @@ pub fn crc24_openpgp_zbc_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies Zbc before selecting this kernel. state = unsafe { crc24_width32_zbc_2way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -927,7 +846,7 @@ pub fn crc24_openpgp_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies Zbc before selecting this kernel. state = unsafe { crc24_width32_zbc_4way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -935,12 +854,8 @@ pub fn crc24_openpgp_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { } /// CRC-24/OPENPGP Zvbc kernel. -/// -/// # Safety -/// -/// Dispatcher verifies Zvbc before selecting this kernel. #[inline] -pub fn crc24_openpgp_zvbc_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_zvbc_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. state = unsafe { crc24_width32_zvbc(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -948,7 +863,7 @@ pub fn crc24_openpgp_zvbc_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. state = unsafe { crc24_width32_zvbc_2way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -956,7 +871,7 @@ pub fn crc24_openpgp_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. state = unsafe { crc24_width32_zvbc_4way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; diff --git a/src/checksum/crc24/s390x.rs b/src/checksum/crc24/s390x.rs index f8b35ca8..9ebe45d3 100644 --- a/src/checksum/crc24/s390x.rs +++ b/src/checksum/crc24/s390x.rs @@ -9,18 +9,15 @@ //! Uses `unsafe` for s390x vector + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, - mem::MaybeUninit, ops::{BitAnd, BitXor, BitXorAssign}, simd::i64x2, }; +use crate::checksum::common::low_u32; + use super::{ keys::{CRC24_OPENPGP_KEYS_REFLECTED, CRC24_OPENPGP_STREAM_REFLECTED}, reflected::{crc24_reflected_update_bitrev_bytes, from_reflected_state, to_reflected_state}, @@ -90,8 +87,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn vgfm(a: i64x2, b: i64x2) -> i64x2 { + fn vgfm(a: i64x2, b: i64x2) -> i64x2 { // SAFETY: Caller guarantees: // 1. VECTOR target features are available (dispatch check). // 2. All SIMD operations are pure register computations after loads. @@ -109,68 +109,62 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let va = Self::new(0, a); - let vb = Self::new(0, b); - Self(Self::vgfm(va.0, vb.0)) - } + fn mul64(a: u64, b: u64) -> Self { + let va = Self::new(0, a); + let vb = Self::new(0, b); + Self(Self::vgfm(va.0, vb.0)) } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_16(self, coeff: Self) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { Self(Self::vgfm(self.0, coeff.swap_lanes().0)) } + fn fold_16(self, coeff: Self) -> Self { + Self(Self::vgfm(self.0, coeff.swap_lanes().0)) } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR + VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { data_to_xor ^ self.fold_16(coeff) } + fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { + data_to_xor ^ self.fold_16(coeff) } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_width32_reflected(self, high: u64, low: u64) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR + VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let clmul = Self::mul64(self.low_64(), low); - let shifted = Self::new(0, self.high_64()); - let mut state = clmul ^ shifted; - - let mask2 = Self::new(0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_0000_0000); - let masked = state & mask2; - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = Self::mul64(shifted_high, high); - state = clmul ^ masked; - - state - } + fn fold_width32_reflected(self, high: u64, low: u64) -> Self { + let clmul = Self::mul64(self.low_64(), low); + let shifted = Self::new(0, self.high_64()); + let mut state = clmul ^ shifted; + + let mask2 = Self::new(0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_0000_0000); + let masked = state & mask2; + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = Self::mul64(shifted_high, high); + state = clmul ^ masked; + + state } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let t1 = Self::mul64(self.low_64(), mu); - let l = Self::mul64(t1.low_64(), poly); - (self ^ l).high_64() as u32 - } + fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { + let t1 = Self::mul64(self.low_64(), mu); + let l = Self::mul64(t1.low_64(), poly); + low_u32((self ^ l).high_64()) } } @@ -178,238 +172,222 @@ impl Simd { #[inline(always)] fn load_block_bitrev(block: &Block) -> [Simd; 8] { - let mut out = MaybeUninit::<[Simd; 8]>::uninit(); - let base = out.as_mut_ptr().cast::(); - - let mut i = 0usize; - while i < 8 { - let low = bitrev_bytes_u64(u64::from_le(block[i * 2])); - let high = bitrev_bytes_u64(u64::from_le(block[i * 2 + 1])); - // SAFETY: `base` points to a `[Simd; 8]` buffer and `i` is in-bounds. - unsafe { - base.add(i).write(Simd::new(high, low)); - } - i = i.strict_add(1); + let mut out = [Simd::new(0, 0); 8]; + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new( + bitrev_bytes_u64(u64::from_le(high)), + bitrev_bytes_u64(u64::from_le(low)), + ); } - - // SAFETY: all 8 elements are initialized above. - unsafe { out.assume_init() } + out } // Folding helpers #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let mut res = x[7]; - res = x[0].fold_16_reflected(Simd::new(keys[10], keys[9]), res); - res = x[1].fold_16_reflected(Simd::new(keys[12], keys[11]), res); - res = x[2].fold_16_reflected(Simd::new(keys[14], keys[13]), res); - res = x[3].fold_16_reflected(Simd::new(keys[16], keys[15]), res); - res = x[4].fold_16_reflected(Simd::new(keys[18], keys[17]), res); - res = x[5].fold_16_reflected(Simd::new(keys[20], keys[19]), res); - res = x[6].fold_16_reflected(Simd::new(keys[2], keys[1]), res); - - res = res.fold_width32_reflected(keys[6], keys[5]); - res.barrett_width32_reflected(keys[8], keys[7]) - } +fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u32 { + let mut res = x[7]; + res = x[0].fold_16_reflected(Simd::new(keys[10], keys[9]), res); + res = x[1].fold_16_reflected(Simd::new(keys[12], keys[11]), res); + res = x[2].fold_16_reflected(Simd::new(keys[14], keys[13]), res); + res = x[3].fold_16_reflected(Simd::new(keys[16], keys[15]), res); + res = x[4].fold_16_reflected(Simd::new(keys[18], keys[17]), res); + res = x[5].fold_16_reflected(Simd::new(keys[20], keys[19]), res); + res = x[6].fold_16_reflected(Simd::new(keys[2], keys[1]), res); + + res = res.fold_width32_reflected(keys[6], keys[5]); + res.barrett_width32_reflected(keys[8], keys[7]) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_width32_reflected_bitrev(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let mut x = load_block_bitrev(first); - x[0] ^= Simd::new(0, state as u64); - - let coeff_128b = Simd::new(keys[4], keys[3]); - for block in rest { - let chunk = load_block_bitrev(block); - x[0] = x[0].fold_16_reflected(coeff_128b, chunk[0]); - x[1] = x[1].fold_16_reflected(coeff_128b, chunk[1]); - x[2] = x[2].fold_16_reflected(coeff_128b, chunk[2]); - x[3] = x[3].fold_16_reflected(coeff_128b, chunk[3]); - x[4] = x[4].fold_16_reflected(coeff_128b, chunk[4]); - x[5] = x[5].fold_16_reflected(coeff_128b, chunk[5]); - x[6] = x[6].fold_16_reflected(coeff_128b, chunk[6]); - x[7] = x[7].fold_16_reflected(coeff_128b, chunk[7]); - } +fn update_simd_width32_reflected_bitrev(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { + let mut x = load_block_bitrev(first); + x[0] ^= Simd::new(0, state as u64); - finalize_lanes_width32_reflected(x, keys) + let coeff_128b = Simd::new(keys[4], keys[3]); + for block in rest { + let chunk = load_block_bitrev(block); + x[0] = x[0].fold_16_reflected(coeff_128b, chunk[0]); + x[1] = x[1].fold_16_reflected(coeff_128b, chunk[1]); + x[2] = x[2].fold_16_reflected(coeff_128b, chunk[2]); + x[3] = x[3].fold_16_reflected(coeff_128b, chunk[3]); + x[4] = x[4].fold_16_reflected(coeff_128b, chunk[4]); + x[5] = x[5].fold_16_reflected(coeff_128b, chunk[5]); + x[6] = x[6].fold_16_reflected(coeff_128b, chunk[6]); + x[7] = x[7].fold_16_reflected(coeff_128b, chunk[7]); } + + finalize_lanes_width32_reflected(x, keys) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn fold_block_128_reflected_bitrev(x: &mut [Simd; 8], block: &Block, coeff: Simd) { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let chunk = load_block_bitrev(block); - x[0] = x[0].fold_16_reflected(coeff, chunk[0]); - x[1] = x[1].fold_16_reflected(coeff, chunk[1]); - x[2] = x[2].fold_16_reflected(coeff, chunk[2]); - x[3] = x[3].fold_16_reflected(coeff, chunk[3]); - x[4] = x[4].fold_16_reflected(coeff, chunk[4]); - x[5] = x[5].fold_16_reflected(coeff, chunk[5]); - x[6] = x[6].fold_16_reflected(coeff, chunk[6]); - x[7] = x[7].fold_16_reflected(coeff, chunk[7]); - } +fn fold_block_128_reflected_bitrev(x: &mut [Simd; 8], block: &Block, coeff: Simd) { + let chunk = load_block_bitrev(block); + x[0] = x[0].fold_16_reflected(coeff, chunk[0]); + x[1] = x[1].fold_16_reflected(coeff, chunk[1]); + x[2] = x[2].fold_16_reflected(coeff, chunk[2]); + x[3] = x[3].fold_16_reflected(coeff, chunk[3]); + x[4] = x[4].fold_16_reflected(coeff, chunk[4]); + x[5] = x[5].fold_16_reflected(coeff, chunk[5]); + x[6] = x[6].fold_16_reflected(coeff, chunk[6]); + x[7] = x[7].fold_16_reflected(coeff, chunk[7]); } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_width32_reflected_bitrev_2way( +fn update_simd_width32_reflected_bitrev_2way( state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_width32_reflected_bitrev(state, first, rest, keys); - } + debug_assert!(!blocks.is_empty()); - let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); - let coeff_128 = Simd::new(keys[4], keys[3]); + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_width32_reflected_bitrev(state, first, rest, keys); + } - let mut s0 = load_block_bitrev(&blocks[0]); - let mut s1 = load_block_bitrev(&blocks[1]); + let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); + let coeff_128 = Simd::new(keys[4], keys[3]); - s0[0] ^= Simd::new(0, state as u64); + let mut s0 = load_block_bitrev(&blocks[0]); + let mut s1 = load_block_bitrev(&blocks[1]); - let mut i: usize = 2; - let even = blocks.len() & !1usize; - while i < even { - fold_block_128_reflected_bitrev(&mut s0, &blocks[i], coeff_256); - fold_block_128_reflected_bitrev(&mut s1, &blocks[i.strict_add(1)], coeff_256); - i = i.strict_add(2); - } + s0[0] ^= Simd::new(0, state as u64); - // Merge: A·s0 ⊕ s1 (A = shift by 128B). - let mut combined = s1; - combined[0] = s0[0].fold_16_reflected(coeff_128, combined[0]); - combined[1] = s0[1].fold_16_reflected(coeff_128, combined[1]); - combined[2] = s0[2].fold_16_reflected(coeff_128, combined[2]); - combined[3] = s0[3].fold_16_reflected(coeff_128, combined[3]); - combined[4] = s0[4].fold_16_reflected(coeff_128, combined[4]); - combined[5] = s0[5].fold_16_reflected(coeff_128, combined[5]); - combined[6] = s0[6].fold_16_reflected(coeff_128, combined[6]); - combined[7] = s0[7].fold_16_reflected(coeff_128, combined[7]); - - if even != blocks.len() { - fold_block_128_reflected_bitrev(&mut combined, &blocks[even], coeff_128); - } + let mut i: usize = 2; + let even = blocks.len() & !1usize; + while i < even { + fold_block_128_reflected_bitrev(&mut s0, &blocks[i], coeff_256); + fold_block_128_reflected_bitrev(&mut s1, &blocks[i.strict_add(1)], coeff_256); + i = i.strict_add(2); + } - finalize_lanes_width32_reflected(combined, keys) + // Merge: A·s0 ⊕ s1 (A = shift by 128B). + let mut combined = s1; + combined[0] = s0[0].fold_16_reflected(coeff_128, combined[0]); + combined[1] = s0[1].fold_16_reflected(coeff_128, combined[1]); + combined[2] = s0[2].fold_16_reflected(coeff_128, combined[2]); + combined[3] = s0[3].fold_16_reflected(coeff_128, combined[3]); + combined[4] = s0[4].fold_16_reflected(coeff_128, combined[4]); + combined[5] = s0[5].fold_16_reflected(coeff_128, combined[5]); + combined[6] = s0[6].fold_16_reflected(coeff_128, combined[6]); + combined[7] = s0[7].fold_16_reflected(coeff_128, combined[7]); + + if even != blocks.len() { + fold_block_128_reflected_bitrev(&mut combined, &blocks[even], coeff_128); } + + finalize_lanes_width32_reflected(combined, keys) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_width32_reflected_bitrev_4way( +fn update_simd_width32_reflected_bitrev_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - debug_assert!(!blocks.is_empty()); + debug_assert!(!blocks.is_empty()); - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_width32_reflected_bitrev(state, first, rest, keys); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_width32_reflected_bitrev(state, first, rest, keys); + } - let aligned = blocks.len().strict_div(4).strict_mul(4); + let aligned = blocks.len().strict_div(4).strict_mul(4); - let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); - let coeff_128 = Simd::new(keys[4], keys[3]); - let c384 = Simd::new(combine[0].0, combine[0].1); - let c256 = Simd::new(combine[1].0, combine[1].1); - let c128 = Simd::new(combine[2].0, combine[2].1); + let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); + let coeff_128 = Simd::new(keys[4], keys[3]); + let c384 = Simd::new(combine[0].0, combine[0].1); + let c256 = Simd::new(combine[1].0, combine[1].1); + let c128 = Simd::new(combine[2].0, combine[2].1); - let mut s0 = load_block_bitrev(&blocks[0]); - let mut s1 = load_block_bitrev(&blocks[1]); - let mut s2 = load_block_bitrev(&blocks[2]); - let mut s3 = load_block_bitrev(&blocks[3]); + let mut s0 = load_block_bitrev(&blocks[0]); + let mut s1 = load_block_bitrev(&blocks[1]); + let mut s2 = load_block_bitrev(&blocks[2]); + let mut s3 = load_block_bitrev(&blocks[3]); - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, state as u64); - let mut i: usize = 4; - while i < aligned { - fold_block_128_reflected_bitrev(&mut s0, &blocks[i], coeff_512); - fold_block_128_reflected_bitrev(&mut s1, &blocks[i.strict_add(1)], coeff_512); - fold_block_128_reflected_bitrev(&mut s2, &blocks[i.strict_add(2)], coeff_512); - fold_block_128_reflected_bitrev(&mut s3, &blocks[i.strict_add(3)], coeff_512); - i = i.strict_add(4); - } - - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - let mut acc = s3; - acc[0] = s2[0].fold_16_reflected(c128, acc[0]); - acc[1] = s2[1].fold_16_reflected(c128, acc[1]); - acc[2] = s2[2].fold_16_reflected(c128, acc[2]); - acc[3] = s2[3].fold_16_reflected(c128, acc[3]); - acc[4] = s2[4].fold_16_reflected(c128, acc[4]); - acc[5] = s2[5].fold_16_reflected(c128, acc[5]); - acc[6] = s2[6].fold_16_reflected(c128, acc[6]); - acc[7] = s2[7].fold_16_reflected(c128, acc[7]); - - acc[0] = s1[0].fold_16_reflected(c256, acc[0]); - acc[1] = s1[1].fold_16_reflected(c256, acc[1]); - acc[2] = s1[2].fold_16_reflected(c256, acc[2]); - acc[3] = s1[3].fold_16_reflected(c256, acc[3]); - acc[4] = s1[4].fold_16_reflected(c256, acc[4]); - acc[5] = s1[5].fold_16_reflected(c256, acc[5]); - acc[6] = s1[6].fold_16_reflected(c256, acc[6]); - acc[7] = s1[7].fold_16_reflected(c256, acc[7]); - - acc[0] = s0[0].fold_16_reflected(c384, acc[0]); - acc[1] = s0[1].fold_16_reflected(c384, acc[1]); - acc[2] = s0[2].fold_16_reflected(c384, acc[2]); - acc[3] = s0[3].fold_16_reflected(c384, acc[3]); - acc[4] = s0[4].fold_16_reflected(c384, acc[4]); - acc[5] = s0[5].fold_16_reflected(c384, acc[5]); - acc[6] = s0[6].fold_16_reflected(c384, acc[6]); - acc[7] = s0[7].fold_16_reflected(c384, acc[7]); - - for block in &blocks[aligned..] { - fold_block_128_reflected_bitrev(&mut acc, block, coeff_128); - } + let mut i: usize = 4; + while i < aligned { + fold_block_128_reflected_bitrev(&mut s0, &blocks[i], coeff_512); + fold_block_128_reflected_bitrev(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128_reflected_bitrev(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128_reflected_bitrev(&mut s3, &blocks[i.strict_add(3)], coeff_512); + i = i.strict_add(4); + } - finalize_lanes_width32_reflected(acc, keys) + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + let mut acc = s3; + acc[0] = s2[0].fold_16_reflected(c128, acc[0]); + acc[1] = s2[1].fold_16_reflected(c128, acc[1]); + acc[2] = s2[2].fold_16_reflected(c128, acc[2]); + acc[3] = s2[3].fold_16_reflected(c128, acc[3]); + acc[4] = s2[4].fold_16_reflected(c128, acc[4]); + acc[5] = s2[5].fold_16_reflected(c128, acc[5]); + acc[6] = s2[6].fold_16_reflected(c128, acc[6]); + acc[7] = s2[7].fold_16_reflected(c128, acc[7]); + + acc[0] = s1[0].fold_16_reflected(c256, acc[0]); + acc[1] = s1[1].fold_16_reflected(c256, acc[1]); + acc[2] = s1[2].fold_16_reflected(c256, acc[2]); + acc[3] = s1[3].fold_16_reflected(c256, acc[3]); + acc[4] = s1[4].fold_16_reflected(c256, acc[4]); + acc[5] = s1[5].fold_16_reflected(c256, acc[5]); + acc[6] = s1[6].fold_16_reflected(c256, acc[6]); + acc[7] = s1[7].fold_16_reflected(c256, acc[7]); + + acc[0] = s0[0].fold_16_reflected(c384, acc[0]); + acc[1] = s0[1].fold_16_reflected(c384, acc[1]); + acc[2] = s0[2].fold_16_reflected(c384, acc[2]); + acc[3] = s0[3].fold_16_reflected(c384, acc[3]); + acc[4] = s0[4].fold_16_reflected(c384, acc[4]); + acc[5] = s0[5].fold_16_reflected(c384, acc[5]); + acc[6] = s0[6].fold_16_reflected(c384, acc[6]); + acc[7] = s0[7].fold_16_reflected(c384, acc[7]); + + for block in &blocks[aligned..] { + fold_block_128_reflected_bitrev(&mut acc, block, coeff_128); } + + finalize_lanes_width32_reflected(acc, keys) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc24_width32_vgfm_bitrev(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. +fn crc24_width32_vgfm_bitrev(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -423,11 +401,13 @@ unsafe fn crc24_width32_vgfm_bitrev(mut state: u32, data: &[u8], keys: &[u64; 23 } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc24_width32_vgfm_bitrev_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. +fn crc24_width32_vgfm_bitrev_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -441,11 +421,13 @@ unsafe fn crc24_width32_vgfm_bitrev_2way(mut state: u32, data: &[u8], keys: &[u6 } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc24_width32_vgfm_bitrev_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. +fn crc24_width32_vgfm_bitrev_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -467,12 +449,8 @@ unsafe fn crc24_width32_vgfm_bitrev_4way(mut state: u32, data: &[u8], keys: &[u6 // Public Safe Kernel /// CRC-24/OPENPGP VGFM kernel. -/// -/// # Safety -/// -/// Dispatcher verifies VECTOR facility before selecting this kernel. #[inline] -pub fn crc24_openpgp_vgfm_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vgfm_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. state = unsafe { crc24_width32_vgfm_bitrev(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -480,7 +458,7 @@ pub fn crc24_openpgp_vgfm_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. state = unsafe { crc24_width32_vgfm_bitrev_2way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -488,7 +466,7 @@ pub fn crc24_openpgp_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. state = unsafe { crc24_width32_vgfm_bitrev_4way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; diff --git a/src/checksum/crc24/x86_64.rs b/src/checksum/crc24/x86_64.rs index 934e4929..e964e39a 100644 --- a/src/checksum/crc24/x86_64.rs +++ b/src/checksum/crc24/x86_64.rs @@ -9,11 +9,9 @@ //! //! # Safety //! -//! Uses `unsafe` for x86 SIMD intrinsics. Callers must ensure SSSE3 + -//! PCLMULQDQ are available before executing these kernels (the dispatcher does -//! this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] +//! The baseline kernels require SSE2, SSSE3, and PCLMULQDQ. The wide kernels +//! additionally require AVX-512F/VL/BW/DQ and VPCLMULQDQ. The private safe +//! wrappers are installed only through capability-gated dispatcher tables. use core::{ arch::x86_64::*, @@ -49,40 +47,77 @@ impl BitXorAssign for Simd128 { } impl Simd128 { + /// Loads 16 bytes without requiring alignment. + /// + /// # Safety + /// + /// `ptr` must be valid to read 16 initialized bytes. The source may be + /// unaligned because the bytes are copied into aligned local storage. + #[inline] + unsafe fn load_unaligned(ptr: *const u8) -> Self { + let mut value = core::mem::MaybeUninit::::uninit(); + + // SAFETY: The caller guarantees a readable 16-byte source. `value` is an + // aligned, non-overlapping 16-byte destination, and every bit pattern is + // valid for the integer vector inside `Simd128`. + unsafe { + core::ptr::copy_nonoverlapping(ptr, value.as_mut_ptr().cast::(), 16); + value.assume_init() + } + } + + /// Creates a vector from its high and low 64-bit lanes. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn new(high: u64, low: u64) -> Self { - // SAFETY: All intrinsics require SSE2, ensured by this function's #[target_feature] attribute. Self(_mm_set_epi64x(high.cast_signed(), low.cast_signed())) } + /// Shifts the vector right by eight bytes, filling the high bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn shift_right_8(self) -> Self { - // SAFETY: All intrinsics require SSE2, ensured by this function's #[target_feature] attribute. Self(_mm_srli_si128::<8>(self.0)) } + /// Shifts the vector left by 12 bytes, filling the low bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn shift_left_12(self) -> Self { - // SAFETY: All intrinsics require SSE2, ensured by this function's #[target_feature] attribute. Self(_mm_slli_si128::<12>(self.0)) } + /// Computes the bitwise AND of two vectors. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn and(self, mask: Self) -> Self { - // SAFETY: All intrinsics require SSE2, ensured by this function's #[target_feature] attribute. Self(_mm_and_si128(self.0, mask.0)) } - /// Reverse bits within each byte (u8::reverse_bits), lane-wise. + /// Reverses the bits within each byte. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and SSSE3. #[inline] #[target_feature(enable = "sse2", enable = "ssse3")] unsafe fn bitrev_bytes(self) -> Self { - // SAFETY: All intrinsics require SSE2+SSSE3, ensured by this function's #[target_feature] - // attribute. let mask = _mm_set1_epi8(0x0f); let lo = _mm_and_si128(self.0, mask); let hi = _mm_and_si128(_mm_srli_epi16::<4>(self.0), mask); @@ -95,17 +130,24 @@ impl Simd128 { Self(_mm_or_si128(lo_shift, hi_rev)) } + /// Folds one reflected 16-byte lane and XORs the supplied input lane. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { - // SAFETY: All intrinsics require SSE2+PCLMULQDQ, ensured by this function's #[target_feature] - // attribute. let h = _mm_clmulepi64_si128::<0x10>(self.0, coeff.0); let l = _mm_clmulepi64_si128::<0x01>(self.0, coeff.0); Self(_mm_xor_si128(_mm_xor_si128(h, l), data_to_xor.0)) } - /// Fold 16 bytes down to the "width32" reduction state (reflected mode). + /// Folds a reflected CRC state from 128 bits to the width-32 reduction state. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_width32_reflected(self, high: u64, low: u64) -> Self { @@ -131,6 +173,11 @@ impl Simd128 { } } + /// Applies Barrett reduction and returns the width-32 state. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { @@ -143,13 +190,18 @@ impl Simd128 { let xorred = _mm_xor_si128(self.0, clmul2); let hi = _mm_srli_si128::<8>(xorred); - _mm_cvtsi128_si64(hi) as u32 + _mm_cvtsi128_si32(hi).cast_unsigned() } } } // 8-lane width32 update (128B blocks) +/// Combines eight folded lanes and applies width-32 Barrett reduction. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn finalize_lanes_width32_reflected(x: [Simd128; 8], keys: &[u64; 23]) -> u32 { @@ -170,6 +222,11 @@ unsafe fn finalize_lanes_width32_reflected(x: [Simd128; 8], keys: &[u64; 23]) -> } } +/// Bit-reverses and folds one or more 128-byte blocks into a width-32 CRC state. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_bitrev_bytes( @@ -221,6 +278,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes( // PCLMULQDQ multi-stream (2/4/7/8-way, 128B blocks) +/// Bit-reverses and folds one 128-byte block into eight parallel lanes. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn fold_block_128_reflected_bitrev(x: &mut [Simd128; 8], chunk: &[Simd128; 8], coeff: Simd128) { @@ -247,6 +309,11 @@ unsafe fn fold_block_128_reflected_bitrev(x: &mut [Simd128; 8], chunk: &[Simd128 } } +/// Bit-reverses and folds 128-byte blocks through two PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_bitrev_bytes_2way( @@ -301,7 +368,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -346,6 +413,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_2way( } } +/// Bit-reverses and folds 128-byte blocks through four PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_bitrev_bytes_4way( @@ -398,7 +470,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_4way( const DOUBLE_GROUP: usize = 8; // 2 × 4-way = 8 blocks = 1KB let mut i: usize = 4; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -422,7 +494,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_4way( } // Handle remaining quads. - let quad_aligned = (blocks.len() / 4) * 4; + let quad_aligned = blocks.len().strict_sub(blocks.len().strict_rem(4)); while i < quad_aligned { fold_block_128_reflected_bitrev(&mut s0, &blocks[i], coeff_512); fold_block_128_reflected_bitrev(&mut s1, &blocks[i.strict_add(1)], coeff_512); @@ -467,6 +539,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_4way( } } +/// Bit-reverses and folds 128-byte blocks through seven PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_bitrev_bytes_7way( @@ -490,7 +567,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_7way( return update_simd_width32_reflected_bitrev_bytes(state, first, rest, keys); } - let aligned = (blocks.len() / 7) * 7; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(7)); let coeff_896 = Simd128::new(fold_896b.0, fold_896b.1); let coeff_128 = Simd128::new(keys[4], keys[3]); @@ -604,6 +681,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_7way( } } +/// Bit-reverses and folds 128-byte blocks through eight PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_bitrev_bytes_8way( @@ -627,7 +709,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_8way( return update_simd_width32_reflected_bitrev_bytes(state, first, rest, keys); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(8)); let coeff_1024 = Simd128::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd128::new(keys[4], keys[3]); @@ -753,6 +835,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_8way( } } +/// Updates CRC-24 with a selected multi-stream PCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn crc24_width32_pclmul_stream( @@ -788,6 +875,11 @@ unsafe fn crc24_width32_pclmul_stream( // Single-stream kernel entry points +/// Updates CRC-24 with the single-stream PCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn crc24_width32_pclmul_small(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { @@ -804,13 +896,13 @@ unsafe fn crc24_width32_pclmul_small(mut state: u32, data: &[u8], keys: &[u64; 2 let coeff_16b = Simd128::new(keys[2], keys[1]); - let mut x0 = Simd128(_mm_loadu_si128(buf as *const __m128i)).bitrev_bytes(); + let mut x0 = Simd128::load_unaligned(buf).bitrev_bytes(); x0 ^= Simd128::new(0, state as u64); buf = buf.add(16); len = len.strict_sub(16); while len >= 16 { - let chunk = Simd128(_mm_loadu_si128(buf as *const __m128i)).bitrev_bytes(); + let chunk = Simd128::load_unaligned(buf).bitrev_bytes(); x0 = x0.fold_16_reflected(coeff_16b, chunk); buf = buf.add(16); len = len.strict_sub(16); @@ -824,6 +916,11 @@ unsafe fn crc24_width32_pclmul_small(mut state: u32, data: &[u8], keys: &[u64; 2 } } +/// Updates CRC-24 with the baseline PCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn crc24_width32_pclmul(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { @@ -845,22 +942,67 @@ unsafe fn crc24_width32_pclmul(mut state: u32, data: &[u8], keys: &[u64; 23]) -> // AVX-512 VPCLMULQDQ Tier +/// Loads 64 bytes without requiring alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 64 initialized bytes. The source may be +/// unaligned because the bytes are copied into aligned local storage. +#[inline] +unsafe fn load_unaligned_512(ptr: *const u8) -> __m512i { + let mut value = core::mem::MaybeUninit::<__m512i>::uninit(); + + // SAFETY: The caller guarantees a readable 64-byte source. `value` is an + // aligned, non-overlapping 64-byte destination, and every bit pattern is + // valid for an integer vector. + unsafe { + core::ptr::copy_nonoverlapping(ptr, value.as_mut_ptr().cast::(), 64); + value.assume_init() + } +} + +/// Stores 64 bytes without requiring alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to write 64 bytes and must not overlap `value`. +#[inline] +unsafe fn store_unaligned_512(ptr: *mut u8, value: __m512i) { + // SAFETY: The caller guarantees a writable, non-overlapping 64-byte + // destination. `value` provides exactly 64 initialized source bytes. + unsafe { core::ptr::copy_nonoverlapping(core::ptr::from_ref(&value).cast::(), ptr, 64) } +} + +/// Multiplies the high lane of each 128-bit element in `a` by the low lane in `b`. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, AVX-512DQ, and +/// VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul10_vpclmul(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: All intrinsics require AVX512F+AVX512VL+AVX512BW+AVX512DQ+VPCLMULQDQ, ensured by this - // function's #[target_feature] attribute. _mm512_clmulepi64_epi128(a, b, 0x10) } +/// Multiplies the low lane of each 128-bit element in `a` by the high lane in `b`. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, AVX-512DQ, and +/// VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul01_vpclmul(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: All intrinsics require AVX512F+AVX512VL+AVX512BW+AVX512DQ+VPCLMULQDQ, ensured by this - // function's #[target_feature] attribute. _mm512_clmulepi64_epi128(a, b, 0x01) } +/// Folds four reflected 16-byte lanes and XORs their supplied input lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, AVX-512DQ, and +/// VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn fold_16_reflected_vpclmul(state: __m512i, coeff: __m512i, data: __m512i) -> __m512i { @@ -870,10 +1012,14 @@ unsafe fn fold_16_reflected_vpclmul(state: __m512i, coeff: __m512i, data: __m512 unsafe { _mm512_ternarylogic_epi64(clmul10_vpclmul(state, coeff), clmul01_vpclmul(state, coeff), data, 0x96) } } +/// Broadcasts a pair of 64-bit coefficients across four 128-bit lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn broadcast_coeff_128b(high: u64, low: u64) -> __m512i { - // SAFETY: All intrinsics require AVX512F, ensured by this function's #[target_feature] attribute. _mm512_set_epi64( high.cast_signed(), low.cast_signed(), @@ -886,18 +1032,25 @@ unsafe fn broadcast_coeff_128b(high: u64, low: u64) -> __m512i { ) } +/// Places the width-32 CRC state in the low 32 bits of lane zero. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn state_mask_lane0(state: u32) -> __m512i { - // SAFETY: All intrinsics require AVX512F, ensured by this function's #[target_feature] attribute. _mm512_set_epi64(0, 0, 0, 0, 0, 0, 0, state as i64) } +/// Reverses the bits within each byte of a 512-bit vector. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F and AVX-512BW. #[inline] #[target_feature(enable = "avx512f,avx512bw")] unsafe fn bitrev_bytes_vpclmul(x: __m512i) -> __m512i { - // SAFETY: All intrinsics require AVX512F+AVX512BW, ensured by this function's #[target_feature] - // attribute. let mask = _mm512_set1_epi8(0x0f); let lo = _mm512_and_si512(x, mask); let hi = _mm512_and_si512(_mm512_srli_epi16(x, 4), mask); @@ -909,6 +1062,11 @@ unsafe fn bitrev_bytes_vpclmul(x: __m512i) -> __m512i { _mm512_or_si512(_mm512_slli_epi16(lo_rev, 4), hi_rev) } +/// Bit-reverses and folds 128-byte blocks into a width-32 VPCLMULQDQ state. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes( @@ -921,9 +1079,9 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes( // AVX512F+AVX512VL+AVX512BW+AVX512DQ+VPCLMULQDQ+SSSE3+PCLMULQDQ+SSE2, ensured by this function's // #[target_feature] attribute. Pointer arithmetic stays within bounds via first/rest structure. unsafe { - let ptr = first.as_ptr() as *const u8; - let mut x0 = _mm512_loadu_si512(ptr as *const __m512i); - let mut x1 = _mm512_loadu_si512(ptr.add(64) as *const __m512i); + let ptr = first.as_ptr().cast::(); + let mut x0 = load_unaligned_512(ptr); + let mut x1 = load_unaligned_512(ptr.add(64)); x0 = bitrev_bytes_vpclmul(x0); x1 = bitrev_bytes_vpclmul(x1); @@ -932,17 +1090,17 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes( let coeff_128b = broadcast_coeff_128b(keys[4], keys[3]); for chunk in rest { - let ptr = chunk.as_ptr() as *const u8; - let y0 = bitrev_bytes_vpclmul(_mm512_loadu_si512(ptr as *const __m512i)); - let y1 = bitrev_bytes_vpclmul(_mm512_loadu_si512(ptr.add(64) as *const __m512i)); + let ptr = chunk.as_ptr().cast::(); + let y0 = bitrev_bytes_vpclmul(load_unaligned_512(ptr)); + let y1 = bitrev_bytes_vpclmul(load_unaligned_512(ptr.add(64))); x0 = fold_16_reflected_vpclmul(x0, coeff_128b, y0); x1 = fold_16_reflected_vpclmul(x1, coeff_128b, y1); } let mut lanes0 = [Simd128(_mm_setzero_si128()); 4]; let mut lanes1 = [Simd128(_mm_setzero_si128()); 4]; - _mm512_storeu_si512(lanes0.as_mut_ptr() as *mut __m512i, x0); - _mm512_storeu_si512(lanes1.as_mut_ptr() as *mut __m512i, x1); + store_unaligned_512(lanes0.as_mut_ptr().cast::(), x0); + store_unaligned_512(lanes1.as_mut_ptr().cast::(), x1); let x = [ lanes0[0], lanes0[1], lanes0[2], lanes0[3], lanes1[0], lanes1[1], lanes1[2], lanes1[3], @@ -952,6 +1110,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes( } } +/// Broadcasts one pair of folding coefficients across four 128-bit lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { @@ -959,6 +1122,11 @@ unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { unsafe { broadcast_coeff_128b(pair.0, pair.1) } } +/// Loads, bit-reverses, and returns one 128-byte block as two vectors. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F and AVX-512BW. #[inline] #[target_feature(enable = "avx512f")] unsafe fn load_128b_block_bitrev(block: &[Simd128; 8]) -> (__m512i, __m512i) { @@ -966,13 +1134,18 @@ unsafe fn load_128b_block_bitrev(block: &[Simd128; 8]) -> (__m512i, __m512i) { // #[target_feature] attribute. Pointer arithmetic stays within bounds: block is 128 bytes, // loading at offsets 0 and 64. unsafe { - let ptr = block.as_ptr() as *const u8; - let y0 = bitrev_bytes_vpclmul(_mm512_loadu_si512(ptr as *const __m512i)); - let y1 = bitrev_bytes_vpclmul(_mm512_loadu_si512(ptr.add(64) as *const __m512i)); + let ptr = block.as_ptr().cast::(); + let y0 = bitrev_bytes_vpclmul(load_unaligned_512(ptr)); + let y1 = bitrev_bytes_vpclmul(load_unaligned_512(ptr.add(64))); (y0, y1) } } +/// Combines two four-lane VPCLMULQDQ states and applies width-32 reduction. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, SSE2, and PCLMULQDQ. #[inline] #[target_feature(enable = "avx512f")] unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i, keys: &[u64; 23]) -> u32 { @@ -999,6 +1172,11 @@ unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i, keys: &[u64; 23]) -> } } +/// Bit-reverses and folds 128-byte blocks through two VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_2way( @@ -1035,7 +1213,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -1091,6 +1269,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_2way( } } +/// Bit-reverses and folds 128-byte blocks through four VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_4way( @@ -1133,7 +1316,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_4way( const DOUBLE_GROUP: usize = 8; // 2 × 4-way = 8 blocks = 1KB let mut i: usize = 4; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -1179,7 +1362,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_4way( } // Handle remaining quads. - let quad_aligned = (blocks.len() / 4) * 4; + let quad_aligned = blocks.len().strict_sub(blocks.len().strict_rem(4)); while i < quad_aligned { let (y0, y1) = load_128b_block_bitrev(&blocks[i]); x0_0 = fold_16_reflected_vpclmul(x0_0, coeff_512, y0); @@ -1217,6 +1400,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_4way( } } +/// Bit-reverses and folds 128-byte blocks through seven VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_7way( @@ -1241,7 +1429,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_7way( return update_simd_width32_reflected_vpclmul_bitrev_bytes(state, first, rest, keys); } - let aligned = (blocks.len() / 7) * 7; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(7)); let (mut x0_0, mut x1_0) = load_128b_block_bitrev(&blocks[0]); let (mut x0_1, mut x1_1) = load_128b_block_bitrev(&blocks[1]); @@ -1325,6 +1513,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_7way( } } +/// Bit-reverses and folds 128-byte blocks through eight VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_8way( @@ -1349,7 +1542,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_8way( return update_simd_width32_reflected_vpclmul_bitrev_bytes(state, first, rest, keys); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(8)); let (mut x0_0, mut x1_0) = load_128b_block_bitrev(&blocks[0]); let (mut x0_1, mut x1_1) = load_128b_block_bitrev(&blocks[1]); @@ -1441,6 +1634,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_8way( } } +/// Updates CRC-24 with a selected multi-stream VPCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn crc24_width32_vpclmul_stream( @@ -1491,6 +1689,11 @@ unsafe fn crc24_width32_vpclmul_stream( } } +/// Updates CRC-24 with the baseline VPCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn crc24_width32_vpclmul(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { @@ -1511,15 +1714,14 @@ unsafe fn crc24_width32_vpclmul(mut state: u32, data: &[u8], keys: &[u64; 23]) - } } -// Public Safe Kernels +// Private safe kernel adapters matching the dispatcher's function signature. /// CRC-24/OPENPGP PCLMULQDQ kernel. /// -/// # Safety -/// -/// Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying SSSE3 and +/// PCLMULQDQ support. #[inline] -pub fn crc24_openpgp_pclmul_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pclmul_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. state = unsafe { crc24_width32_pclmul(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -1530,11 +1732,10 @@ pub fn crc24_openpgp_pclmul_safe(crc: u32, data: &[u8]) -> u32 { /// /// Optimized for inputs smaller than a folding block (128 bytes). /// -/// # Safety -/// -/// Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying SSSE3 and +/// PCLMULQDQ support. #[inline] -pub fn crc24_openpgp_pclmul_small_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pclmul_small_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. state = unsafe { crc24_width32_pclmul_small(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -1543,7 +1744,7 @@ pub fn crc24_openpgp_pclmul_small_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP PCLMULQDQ kernel (2-way multi-stream). #[inline] -pub fn crc24_openpgp_pclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. state = unsafe { @@ -1560,7 +1761,7 @@ pub fn crc24_openpgp_pclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP PCLMULQDQ kernel (4-way multi-stream). #[inline] -pub fn crc24_openpgp_pclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. state = unsafe { @@ -1577,7 +1778,7 @@ pub fn crc24_openpgp_pclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP PCLMULQDQ kernel (7-way multi-stream). #[inline] -pub fn crc24_openpgp_pclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. state = unsafe { @@ -1594,7 +1795,7 @@ pub fn crc24_openpgp_pclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP PCLMULQDQ kernel (8-way multi-stream). #[inline] -pub fn crc24_openpgp_pclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. state = unsafe { @@ -1611,11 +1812,10 @@ pub fn crc24_openpgp_pclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP VPCLMULQDQ kernel (AVX-512). /// -/// # Safety -/// -/// Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying VPCLMULQDQ +/// and AVX-512 support. #[inline] -pub fn crc24_openpgp_vpclmul_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpclmul_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. state = unsafe { crc24_width32_vpclmul(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -1624,7 +1824,7 @@ pub fn crc24_openpgp_vpclmul_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP VPCLMULQDQ kernel (2-way multi-stream). #[inline] -pub fn crc24_openpgp_vpclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. state = unsafe { @@ -1641,7 +1841,7 @@ pub fn crc24_openpgp_vpclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP VPCLMULQDQ kernel (4-way multi-stream). #[inline] -pub fn crc24_openpgp_vpclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. state = unsafe { @@ -1658,7 +1858,7 @@ pub fn crc24_openpgp_vpclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP VPCLMULQDQ kernel (7-way multi-stream). #[inline] -pub fn crc24_openpgp_vpclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. state = unsafe { @@ -1675,7 +1875,7 @@ pub fn crc24_openpgp_vpclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP VPCLMULQDQ kernel (8-way multi-stream). #[inline] -pub fn crc24_openpgp_vpclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. state = unsafe { @@ -1702,14 +1902,21 @@ mod tests { const OFFSETS: &[usize] = &[0, 1, 7, 15]; const STATES: &[u32] = &[0, 0x00b7_04ce, 0x005a_a5a5, 0x00ff_ffff]; + fn data() -> Vec { + (0u16..4111) + .map(|i| { + let [low, high] = i.to_le_bytes(); + low.wrapping_mul(61).wrapping_add(high) + }) + .collect() + } + fn assert_kernel(name: &str, kernel: fn(u32, &[u8]) -> u32) { - let input: Vec = (0..4111) - .map(|i| (i as u8).wrapping_mul(61).wrapping_add((i >> 8) as u8)) - .collect(); + let input = data(); for &state in STATES { for &offset in OFFSETS { for &len in LENS { - let slice = &input[offset..offset + len]; + let slice = &input[offset..offset.strict_add(len)]; assert_eq!( kernel(state, slice), super::super::portable::crc24_openpgp_slice8(state, slice), diff --git a/src/checksum/crc32/aarch64.rs b/src/checksum/crc32/aarch64.rs index 19703708..f594b125 100644 --- a/src/checksum/crc32/aarch64.rs +++ b/src/checksum/crc32/aarch64.rs @@ -10,12 +10,9 @@ //! Uses `unsafe` for aarch64 intrinsics. Callers must ensure required target //! features are available before selecting a kernel (the dispatcher does this). -#![allow(unsafe_code)] -#![cfg_attr(not(any(target_os = "linux", target_os = "android")), allow(dead_code))] // SAFETY: This module is intrinsics-heavy and uses tight, invariant-driven indexing // (e.g. fixed-size lanes and chunked processing) where bounds are proven by // control flow; Clippy cannot always see these invariants. -#![allow(clippy::indexing_slicing)] // This module is intrinsics-heavy; unsafe blocks are per-function with SAFETY justifications. use core::{arch::aarch64::*, ptr}; @@ -28,11 +25,64 @@ use crate::checksum::common::{ const CRC32_SHIFT8_MATRIX: Gf2Matrix32 = generate_shift8_matrix_32(CRC32_IEEE_POLY); const CRC32C_SHIFT8_MATRIX: Gf2Matrix32 = generate_shift8_matrix_32(CRC32C_POLY); +/// Read an integer from an unaligned byte pointer. +/// +/// # Safety +/// +/// `source` must remain valid to read two initialized bytes from one live allocation. +#[inline(always)] +unsafe fn load_u16(source: *const u8) -> u16 { + // SAFETY: The caller provides the validity and initialization contract; `read_unaligned` does + // not require `source` to satisfy `u16` alignment. + unsafe { ptr::read_unaligned(source.cast()) } +} + +/// Read an integer from an unaligned byte pointer. +/// +/// # Safety +/// +/// `source` must remain valid to read four initialized bytes from one live allocation. +#[inline(always)] +unsafe fn load_u32(source: *const u8) -> u32 { + // SAFETY: The caller provides the validity and initialization contract; `read_unaligned` does + // not require `source` to satisfy `u32` alignment. + unsafe { ptr::read_unaligned(source.cast()) } +} + +/// Read an integer from an unaligned byte pointer. +/// +/// # Safety +/// +/// `source` must remain valid to read eight initialized bytes from one live allocation. +#[inline(always)] +unsafe fn load_u64(source: *const u8) -> u64 { + // SAFETY: The caller provides the validity and initialization contract; `read_unaligned` does + // not require `source` to satisfy `u64` alignment. + unsafe { ptr::read_unaligned(source.cast()) } +} + +/// Load two `u64` lanes from an unaligned byte pointer. +/// +/// # Safety +/// +/// `source` must remain valid to read 16 initialized bytes from one live allocation. The caller +/// must also ensure the AArch64 NEON feature is available. +#[inline(always)] +unsafe fn load_u64x2(source: *const u8) -> uint64x2_t { + // SAFETY: The caller provides the memory and target-feature contracts. `vld1q_u64` implements + // its load with `read_unaligned`, so `source` need not satisfy `u64` alignment. + unsafe { vld1q_u64(source.cast()) } +} + // Hardware CRC extension (CRC-only) /// CRC-32 (IEEE) update using ARMv8 CRC extension. /// /// `crc` is the current state (pre-inverted). +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC target feature is available. #[inline] #[target_feature(enable = "crc")] unsafe fn crc32_armv8(crc: u32, data: &[u8]) -> u32 { @@ -53,41 +103,41 @@ unsafe fn crc32_armv8(crc: u32, data: &[u8]) -> u32 { } while len >= 64 { - state = __crc32d(state, ptr::read_unaligned(buf as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(8) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(16) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(24) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(32) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(40) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(48) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(56) as *const u64)); + state = __crc32d(state, load_u64(buf)); + state = __crc32d(state, load_u64(buf.add(8))); + state = __crc32d(state, load_u64(buf.add(16))); + state = __crc32d(state, load_u64(buf.add(24))); + state = __crc32d(state, load_u64(buf.add(32))); + state = __crc32d(state, load_u64(buf.add(40))); + state = __crc32d(state, load_u64(buf.add(48))); + state = __crc32d(state, load_u64(buf.add(56))); buf = buf.add(64); len = len.strict_sub(64); } while len >= 32 { - state = __crc32d(state, ptr::read_unaligned(buf as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(8) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(16) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(24) as *const u64)); + state = __crc32d(state, load_u64(buf)); + state = __crc32d(state, load_u64(buf.add(8))); + state = __crc32d(state, load_u64(buf.add(16))); + state = __crc32d(state, load_u64(buf.add(24))); buf = buf.add(32); len = len.strict_sub(32); } while len >= 8 { - state = __crc32d(state, ptr::read_unaligned(buf as *const u64)); + state = __crc32d(state, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } if len >= 4 { - state = __crc32w(state, ptr::read_unaligned(buf as *const u32)); + state = __crc32w(state, load_u32(buf)); buf = buf.add(4); len = len.strict_sub(4); } if len >= 2 { - state = __crc32h(state, ptr::read_unaligned(buf as *const u16)); + state = __crc32h(state, load_u16(buf)); buf = buf.add(2); len = len.strict_sub(2); } @@ -103,6 +153,10 @@ unsafe fn crc32_armv8(crc: u32, data: &[u8]) -> u32 { /// CRC-32C (Castagnoli) update using ARMv8 CRC extension. /// /// `crc` is the current state (pre-inverted). +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC target feature is available. #[inline] #[target_feature(enable = "crc")] unsafe fn crc32c_armv8(crc: u32, data: &[u8]) -> u32 { @@ -123,41 +177,41 @@ unsafe fn crc32c_armv8(crc: u32, data: &[u8]) -> u32 { } while len >= 64 { - state = __crc32cd(state, ptr::read_unaligned(buf as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(8) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(16) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(24) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(32) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(40) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(48) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(56) as *const u64)); + state = __crc32cd(state, load_u64(buf)); + state = __crc32cd(state, load_u64(buf.add(8))); + state = __crc32cd(state, load_u64(buf.add(16))); + state = __crc32cd(state, load_u64(buf.add(24))); + state = __crc32cd(state, load_u64(buf.add(32))); + state = __crc32cd(state, load_u64(buf.add(40))); + state = __crc32cd(state, load_u64(buf.add(48))); + state = __crc32cd(state, load_u64(buf.add(56))); buf = buf.add(64); len = len.strict_sub(64); } while len >= 32 { - state = __crc32cd(state, ptr::read_unaligned(buf as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(8) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(16) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(24) as *const u64)); + state = __crc32cd(state, load_u64(buf)); + state = __crc32cd(state, load_u64(buf.add(8))); + state = __crc32cd(state, load_u64(buf.add(16))); + state = __crc32cd(state, load_u64(buf.add(24))); buf = buf.add(32); len = len.strict_sub(32); } while len >= 8 { - state = __crc32cd(state, ptr::read_unaligned(buf as *const u64)); + state = __crc32cd(state, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } if len >= 4 { - state = __crc32cw(state, ptr::read_unaligned(buf as *const u32)); + state = __crc32cw(state, load_u32(buf)); buf = buf.add(4); len = len.strict_sub(4); } if len >= 2 { - state = __crc32ch(state, ptr::read_unaligned(buf as *const u16)); + state = __crc32ch(state, load_u16(buf)); buf = buf.add(2); len = len.strict_sub(2); } @@ -172,20 +226,26 @@ unsafe fn crc32c_armv8(crc: u32, data: &[u8]) -> u32 { /// Safe wrapper for CRC-32 ARMv8 CRC extension kernel. #[inline] -pub fn crc32_armv8_safe(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32_armv8_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC extension before selecting this kernel. unsafe { crc32_armv8(crc, data) } } /// Safe wrapper for CRC-32C ARMv8 CRC extension kernel. #[inline] -pub fn crc32c_armv8_safe(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32c_armv8_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC extension before selecting this kernel. unsafe { crc32c_armv8(crc, data) } } // Hardware CRC extension (multi-stream wrappers) +/// Update CRC-32 (IEEE) through `N` independent ARMv8 CRC streams. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC target feature is available and instantiate `N` as two +/// or three. #[inline] #[target_feature(enable = "crc")] unsafe fn crc32_armv8_nway(crc: u32, data: &[u8]) -> u32 { @@ -201,7 +261,7 @@ unsafe fn crc32_armv8_nway(crc: u32, data: &[u8]) -> u32 { return crc32_armv8(crc, data); } - let chunk_len = len / N; + let chunk_len = len.strict_div(N); let mut lanes = [!0u32; N]; let mut i: usize = 0; @@ -210,10 +270,7 @@ unsafe fn crc32_armv8_nway(crc: u32, data: &[u8]) -> u32 { let mut lane_idx: usize = 0; while lane_idx < N { let base = lane_idx.strict_mul(chunk_len).strict_add(i); - lanes[lane_idx] = __crc32d( - lanes[lane_idx], - ptr::read_unaligned(data.as_ptr().add(base) as *const u64), - ); + lanes[lane_idx] = __crc32d(lanes[lane_idx], load_u64(data.as_ptr().add(base))); lane_idx = lane_idx.strict_add(1); } i = i.strict_add(8); @@ -230,7 +287,7 @@ unsafe fn crc32_armv8_nway(crc: u32, data: &[u8]) -> u32 { let tail_start = chunk_len.strict_mul(N); if tail_start < len { - lanes[N - 1] = crc32_armv8(lanes[N - 1], data.get_unchecked(tail_start..)); + lanes[N.strict_sub(1)] = crc32_armv8(lanes[N.strict_sub(1)], data.get_unchecked(tail_start..)); } let mut data_crc_final: u32 = 0; @@ -257,6 +314,12 @@ unsafe fn crc32_armv8_nway(crc: u32, data: &[u8]) -> u32 { } } +/// Update CRC-32C (Castagnoli) through `N` independent ARMv8 CRC streams. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC target feature is available and instantiate `N` as two +/// or three. #[inline] #[target_feature(enable = "crc")] unsafe fn crc32c_armv8_nway(crc: u32, data: &[u8]) -> u32 { @@ -272,7 +335,7 @@ unsafe fn crc32c_armv8_nway(crc: u32, data: &[u8]) -> u32 { return crc32c_armv8(crc, data); } - let chunk_len = len / N; + let chunk_len = len.strict_div(N); let mut lanes = [!0u32; N]; let mut i: usize = 0; @@ -281,10 +344,7 @@ unsafe fn crc32c_armv8_nway(crc: u32, data: &[u8]) -> u32 { let mut lane_idx: usize = 0; while lane_idx < N { let base = lane_idx.strict_mul(chunk_len).strict_add(i); - lanes[lane_idx] = __crc32cd( - lanes[lane_idx], - ptr::read_unaligned(data.as_ptr().add(base) as *const u64), - ); + lanes[lane_idx] = __crc32cd(lanes[lane_idx], load_u64(data.as_ptr().add(base))); lane_idx = lane_idx.strict_add(1); } i = i.strict_add(8); @@ -301,7 +361,7 @@ unsafe fn crc32c_armv8_nway(crc: u32, data: &[u8]) -> u32 { let tail_start = chunk_len.strict_mul(N); if tail_start < len { - lanes[N - 1] = crc32c_armv8(lanes[N - 1], data.get_unchecked(tail_start..)); + lanes[N.strict_sub(1)] = crc32c_armv8(lanes[N.strict_sub(1)], data.get_unchecked(tail_start..)); } let mut data_crc_final: u32 = 0; @@ -329,25 +389,25 @@ unsafe fn crc32c_armv8_nway(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_armv8_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_armv8_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC extension before selecting this kernel. unsafe { crc32_armv8_nway::<2>(crc, data) } } #[inline] -pub fn crc32_armv8_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_armv8_3way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC extension before selecting this kernel. unsafe { crc32_armv8_nway::<3>(crc, data) } } #[inline] -pub fn crc32c_armv8_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_armv8_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC extension before selecting this kernel. unsafe { crc32c_armv8_nway::<2>(crc, data) } } #[inline] -pub fn crc32c_armv8_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_armv8_3way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC extension before selecting this kernel. unsafe { crc32c_armv8_nway::<3>(crc, data) } } @@ -391,7 +451,7 @@ fn crc32_sve2_pmull_nway( let tail_start = chunk_len.strict_mul(N); if tail_start < len { - lanes[N - 1] = update(lanes[N - 1], &data[tail_start..]); + lanes[N.strict_sub(1)] = update(lanes[N.strict_sub(1)], &data[tail_start..]); } // Combine the N independent CRCs with precomputed matrices to avoid @@ -422,51 +482,55 @@ fn crc32_sve2_pmull_nway( } #[inline] -pub fn crc32_iso_hdlc_sve2_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc32_iso_hdlc_sve2_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<2>(crc, data, crc32_iso_hdlc_pmull_v12e_v1_safe, CRC32_SHIFT8_MATRIX) } #[inline] -pub fn crc32_iso_hdlc_sve2_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc32_iso_hdlc_sve2_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<3>(crc, data, crc32_iso_hdlc_pmull_v12e_v1_safe, CRC32_SHIFT8_MATRIX) } #[inline] -pub fn crc32c_iscsi_sve2_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc32c_iscsi_sve2_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<2>(crc, data, crc32c_iscsi_pmull_v12e_v1_safe, CRC32C_SHIFT8_MATRIX) } #[inline] -pub fn crc32c_iscsi_sve2_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc32c_iscsi_sve2_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<3>(crc, data, crc32c_iscsi_pmull_v12e_v1_safe, CRC32C_SHIFT8_MATRIX) } // PMULL tier multi-stream wrappers (2/3-way) #[inline] -pub fn crc32_iso_hdlc_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<2>(crc, data, crc32_iso_hdlc_pmull_v9s3x2e_s3_safe, CRC32_SHIFT8_MATRIX) } #[inline] -pub fn crc32_iso_hdlc_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<3>(crc, data, crc32_iso_hdlc_pmull_v9s3x2e_s3_safe, CRC32_SHIFT8_MATRIX) } #[inline] -pub fn crc32c_iscsi_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<2>(crc, data, crc32c_iscsi_pmull_v9s3x2e_s3_safe, CRC32C_SHIFT8_MATRIX) } #[inline] -pub fn crc32c_iscsi_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<3>(crc, data, crc32c_iscsi_pmull_v9s3x2e_s3_safe, CRC32C_SHIFT8_MATRIX) } // EOR3 tier multi-stream wrappers (2/3-way) #[inline] -pub fn crc32_iso_hdlc_pmull_eor3_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_eor3_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<2>( crc, data, @@ -476,7 +540,7 @@ pub fn crc32_iso_hdlc_pmull_eor3_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_iso_hdlc_pmull_eor3_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_eor3_3way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<3>( crc, data, @@ -486,19 +550,19 @@ pub fn crc32_iso_hdlc_pmull_eor3_3way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32c_iscsi_pmull_eor3_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_eor3_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<2>(crc, data, crc32c_iscsi_pmull_eor3_v9s3x2e_s3_safe, CRC32C_SHIFT8_MATRIX) } #[inline] -pub fn crc32c_iscsi_pmull_eor3_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_eor3_3way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<3>(crc, data, crc32c_iscsi_pmull_eor3_v9s3x2e_s3_safe, CRC32C_SHIFT8_MATRIX) } // Small-buffer wrappers (selected for len < fold block) #[inline] -pub fn crc32_iso_hdlc_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { if data.len() <= 256 { return crc32_armv8_safe(crc, data); } @@ -506,7 +570,7 @@ pub fn crc32_iso_hdlc_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32c_iscsi_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { if data.len() <= 256 { return crc32c_armv8_safe(crc, data); } @@ -514,12 +578,14 @@ pub fn crc32c_iscsi_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_iso_hdlc_sve2_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc32_iso_hdlc_sve2_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { crc32_iso_hdlc_pmull_v12e_v1_safe(crc, data) } #[inline] -pub fn crc32c_iscsi_sve2_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc32c_iscsi_sve2_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { crc32c_iscsi_pmull_v12e_v1_safe(crc, data) } @@ -533,7 +599,7 @@ pub fn crc32c_iscsi_sve2_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { /// /// # Safety /// -/// Caller must verify that both CRC and AES (PMLL) target features are +/// Caller must verify that both CRC and AES (PMULL) target features are /// available at runtime (via capability detection). #[inline] #[target_feature(enable = "crc,aes")] @@ -546,7 +612,7 @@ pub(crate) unsafe fn crc32_ieee_fusion_inline(crc: u32, data: &[u8]) -> u32 { /// /// # Safety /// -/// Caller must verify that both CRC and AES (PMLL) target features are +/// Caller must verify that both CRC and AES (PMULL) target features are /// available at runtime (via capability detection). #[inline] #[target_feature(enable = "crc,aes")] @@ -585,31 +651,47 @@ pub(crate) unsafe fn crc32c_iscsi_hwcrc_inline(crc: u32, data: &[u8]) -> u32 { // PMULL helpers (fusion kernels) +/// Carry-less multiply the low lanes of two vectors. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 AES/PMULL target feature is available. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul_lo(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { - // SAFETY: Caller guarantees NEON+AES (PMULL) is available. - // vmull_p64, vgetq_lane_u64, vreinterpretq_u64_p128 are safe with target_feature on the fn. let result = vmull_p64(vgetq_lane_u64(a, 0), vgetq_lane_u64(b, 0)); vreinterpretq_u64_p128(result) } +/// Carry-less multiply the high lanes of two vectors. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 AES/PMULL target feature is available. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul_hi(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { - // SAFETY: Caller guarantees NEON+AES (PMULL) is available. let result = vmull_p64(vgetq_lane_u64(a, 1), vgetq_lane_u64(b, 1)); vreinterpretq_u64_p128(result) } +/// Carry-less multiply two scalar CRC values. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 AES/PMULL target feature is available. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul_scalar(a: u32, b: u32) -> uint64x2_t { - // SAFETY: Caller guarantees NEON+AES (PMULL) is available. let result = vmull_p64(a as u64, b as u64); vreinterpretq_u64_p128(result) } +/// Carry-less multiply the low lanes, then XOR the product with `c`. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 AES/PMULL target feature is available. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul_lo_and_xor(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { @@ -617,6 +699,11 @@ unsafe fn clmul_lo_and_xor(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint6 unsafe { veorq_u64(clmul_lo(a, b), c) } } +/// Carry-less multiply the high lanes, then XOR the product with `c`. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 AES/PMULL target feature is available. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul_hi_and_xor(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { @@ -626,6 +713,12 @@ unsafe fn clmul_hi_and_xor(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint6 // Fusion: CRC-32C (iSCSI) - PMULL v12e_v1 +/// Update CRC-32C with the 12-stream PMULL fusion kernel. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. When `len` +/// is nonzero, `buf` must be valid to read `len` initialized bytes from one allocation. #[inline] #[target_feature(enable = "crc,aes")] unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -641,7 +734,7 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: } if (buf as usize & 8) != 0 && len >= 8 { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -650,18 +743,18 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: let end = buf.add(len); let limit = buf.add(len.strict_sub(192)); - let mut x0 = vld1q_u64(buf as *const u64); - let mut x1 = vld1q_u64(buf.add(16) as *const u64); - let mut x2 = vld1q_u64(buf.add(32) as *const u64); - let mut x3 = vld1q_u64(buf.add(48) as *const u64); - let mut x4 = vld1q_u64(buf.add(64) as *const u64); - let mut x5 = vld1q_u64(buf.add(80) as *const u64); - let mut x6 = vld1q_u64(buf.add(96) as *const u64); - let mut x7 = vld1q_u64(buf.add(112) as *const u64); - let mut x8 = vld1q_u64(buf.add(128) as *const u64); - let mut x9 = vld1q_u64(buf.add(144) as *const u64); - let mut x10 = vld1q_u64(buf.add(160) as *const u64); - let mut x11 = vld1q_u64(buf.add(176) as *const u64); + let mut x0 = load_u64x2(buf); + let mut x1 = load_u64x2(buf.add(16)); + let mut x2 = load_u64x2(buf.add(32)); + let mut x3 = load_u64x2(buf.add(48)); + let mut x4 = load_u64x2(buf.add(64)); + let mut x5 = load_u64x2(buf.add(80)); + let mut x6 = load_u64x2(buf.add(96)); + let mut x7 = load_u64x2(buf.add(112)); + let mut x8 = load_u64x2(buf.add(128)); + let mut x9 = load_u64x2(buf.add(144)); + let mut x10 = load_u64x2(buf.add(160)); + let mut x11 = load_u64x2(buf.add(176)); let k_vals: [u64; 2] = [0xa87ab8a8, 0xab7aff2a]; let mut k = vld1q_u64(k_vals.as_ptr()); @@ -671,29 +764,29 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: buf = buf.add(192); while buf <= limit { - let y0 = clmul_lo_and_xor(x0, k, vld1q_u64(buf as *const u64)); + let y0 = clmul_lo_and_xor(x0, k, load_u64x2(buf)); x0 = clmul_hi_and_xor(x0, k, y0); - let y1 = clmul_lo_and_xor(x1, k, vld1q_u64(buf.add(16) as *const u64)); + let y1 = clmul_lo_and_xor(x1, k, load_u64x2(buf.add(16))); x1 = clmul_hi_and_xor(x1, k, y1); - let y2 = clmul_lo_and_xor(x2, k, vld1q_u64(buf.add(32) as *const u64)); + let y2 = clmul_lo_and_xor(x2, k, load_u64x2(buf.add(32))); x2 = clmul_hi_and_xor(x2, k, y2); - let y3 = clmul_lo_and_xor(x3, k, vld1q_u64(buf.add(48) as *const u64)); + let y3 = clmul_lo_and_xor(x3, k, load_u64x2(buf.add(48))); x3 = clmul_hi_and_xor(x3, k, y3); - let y4 = clmul_lo_and_xor(x4, k, vld1q_u64(buf.add(64) as *const u64)); + let y4 = clmul_lo_and_xor(x4, k, load_u64x2(buf.add(64))); x4 = clmul_hi_and_xor(x4, k, y4); - let y5 = clmul_lo_and_xor(x5, k, vld1q_u64(buf.add(80) as *const u64)); + let y5 = clmul_lo_and_xor(x5, k, load_u64x2(buf.add(80))); x5 = clmul_hi_and_xor(x5, k, y5); - let y6 = clmul_lo_and_xor(x6, k, vld1q_u64(buf.add(96) as *const u64)); + let y6 = clmul_lo_and_xor(x6, k, load_u64x2(buf.add(96))); x6 = clmul_hi_and_xor(x6, k, y6); - let y7 = clmul_lo_and_xor(x7, k, vld1q_u64(buf.add(112) as *const u64)); + let y7 = clmul_lo_and_xor(x7, k, load_u64x2(buf.add(112))); x7 = clmul_hi_and_xor(x7, k, y7); - let y8 = clmul_lo_and_xor(x8, k, vld1q_u64(buf.add(128) as *const u64)); + let y8 = clmul_lo_and_xor(x8, k, load_u64x2(buf.add(128))); x8 = clmul_hi_and_xor(x8, k, y8); - let y9 = clmul_lo_and_xor(x9, k, vld1q_u64(buf.add(144) as *const u64)); + let y9 = clmul_lo_and_xor(x9, k, load_u64x2(buf.add(144))); x9 = clmul_hi_and_xor(x9, k, y9); - let y10 = clmul_lo_and_xor(x10, k, vld1q_u64(buf.add(160) as *const u64)); + let y10 = clmul_lo_and_xor(x10, k, load_u64x2(buf.add(160))); x10 = clmul_hi_and_xor(x10, k, y10); - let y11 = clmul_lo_and_xor(x11, k, vld1q_u64(buf.add(176) as *const u64)); + let y11 = clmul_lo_and_xor(x11, k, load_u64x2(buf.add(176))); x11 = clmul_hi_and_xor(x11, k, y11); buf = buf.add(192); } @@ -735,11 +828,11 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: crc0 = __crc32cd(0, vgetq_lane_u64(x0, 0)); crc0 = __crc32cd(crc0, vgetq_lane_u64(x0, 1)); - len = end.offset_from(buf) as usize; + len = end.offset_from_unsigned(buf); } if len >= 16 { - let mut x0 = vld1q_u64(buf as *const u64); + let mut x0 = load_u64x2(buf); let k_vals: [u64; 2] = [0xf20c0dfe, 0x493c7d27]; let k = vld1q_u64(k_vals.as_ptr()); @@ -750,7 +843,7 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: len = len.strict_sub(16); while len >= 16 { - let y0 = clmul_lo_and_xor(x0, k, vld1q_u64(buf as *const u64)); + let y0 = clmul_lo_and_xor(x0, k, load_u64x2(buf)); x0 = clmul_hi_and_xor(x0, k, y0); buf = buf.add(16); len = len.strict_sub(16); @@ -761,7 +854,7 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: } while len >= 8 { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -778,13 +871,19 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: /// Safe wrapper for CRC-32C fusion kernel (CRC+PMULL v12e_v1). #[inline] -pub fn crc32c_iscsi_pmull_v12e_v1_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_v12e_v1_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC + PMULL before selecting this kernel. unsafe { crc32c_iscsi_pmull_v12e_v1(crc, data.as_ptr(), data.len()) } } // Fusion: CRC-32 (ISO-HDLC / IEEE) - PMULL v12e_v1 +/// Update CRC-32 (IEEE) with the 12-stream PMULL fusion kernel. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. When `len` +/// is nonzero, `buf` must be valid to read `len` initialized bytes from one allocation. #[inline] #[target_feature(enable = "crc,aes")] unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -800,7 +899,7 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le } if (buf as usize & 8) != 0 && len >= 8 { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -809,18 +908,18 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le let end = buf.add(len); let limit = buf.add(len.strict_sub(192)); - let mut x0 = vld1q_u64(buf as *const u64); - let mut x1 = vld1q_u64(buf.add(16) as *const u64); - let mut x2 = vld1q_u64(buf.add(32) as *const u64); - let mut x3 = vld1q_u64(buf.add(48) as *const u64); - let mut x4 = vld1q_u64(buf.add(64) as *const u64); - let mut x5 = vld1q_u64(buf.add(80) as *const u64); - let mut x6 = vld1q_u64(buf.add(96) as *const u64); - let mut x7 = vld1q_u64(buf.add(112) as *const u64); - let mut x8 = vld1q_u64(buf.add(128) as *const u64); - let mut x9 = vld1q_u64(buf.add(144) as *const u64); - let mut x10 = vld1q_u64(buf.add(160) as *const u64); - let mut x11 = vld1q_u64(buf.add(176) as *const u64); + let mut x0 = load_u64x2(buf); + let mut x1 = load_u64x2(buf.add(16)); + let mut x2 = load_u64x2(buf.add(32)); + let mut x3 = load_u64x2(buf.add(48)); + let mut x4 = load_u64x2(buf.add(64)); + let mut x5 = load_u64x2(buf.add(80)); + let mut x6 = load_u64x2(buf.add(96)); + let mut x7 = load_u64x2(buf.add(112)); + let mut x8 = load_u64x2(buf.add(128)); + let mut x9 = load_u64x2(buf.add(144)); + let mut x10 = load_u64x2(buf.add(160)); + let mut x11 = load_u64x2(buf.add(176)); let k_vals: [u64; 2] = [0x596c8d81, 0xf5e48c85]; let mut k = vld1q_u64(k_vals.as_ptr()); @@ -830,29 +929,29 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le buf = buf.add(192); while buf <= limit { - let y0 = clmul_lo_and_xor(x0, k, vld1q_u64(buf as *const u64)); + let y0 = clmul_lo_and_xor(x0, k, load_u64x2(buf)); x0 = clmul_hi_and_xor(x0, k, y0); - let y1 = clmul_lo_and_xor(x1, k, vld1q_u64(buf.add(16) as *const u64)); + let y1 = clmul_lo_and_xor(x1, k, load_u64x2(buf.add(16))); x1 = clmul_hi_and_xor(x1, k, y1); - let y2 = clmul_lo_and_xor(x2, k, vld1q_u64(buf.add(32) as *const u64)); + let y2 = clmul_lo_and_xor(x2, k, load_u64x2(buf.add(32))); x2 = clmul_hi_and_xor(x2, k, y2); - let y3 = clmul_lo_and_xor(x3, k, vld1q_u64(buf.add(48) as *const u64)); + let y3 = clmul_lo_and_xor(x3, k, load_u64x2(buf.add(48))); x3 = clmul_hi_and_xor(x3, k, y3); - let y4 = clmul_lo_and_xor(x4, k, vld1q_u64(buf.add(64) as *const u64)); + let y4 = clmul_lo_and_xor(x4, k, load_u64x2(buf.add(64))); x4 = clmul_hi_and_xor(x4, k, y4); - let y5 = clmul_lo_and_xor(x5, k, vld1q_u64(buf.add(80) as *const u64)); + let y5 = clmul_lo_and_xor(x5, k, load_u64x2(buf.add(80))); x5 = clmul_hi_and_xor(x5, k, y5); - let y6 = clmul_lo_and_xor(x6, k, vld1q_u64(buf.add(96) as *const u64)); + let y6 = clmul_lo_and_xor(x6, k, load_u64x2(buf.add(96))); x6 = clmul_hi_and_xor(x6, k, y6); - let y7 = clmul_lo_and_xor(x7, k, vld1q_u64(buf.add(112) as *const u64)); + let y7 = clmul_lo_and_xor(x7, k, load_u64x2(buf.add(112))); x7 = clmul_hi_and_xor(x7, k, y7); - let y8 = clmul_lo_and_xor(x8, k, vld1q_u64(buf.add(128) as *const u64)); + let y8 = clmul_lo_and_xor(x8, k, load_u64x2(buf.add(128))); x8 = clmul_hi_and_xor(x8, k, y8); - let y9 = clmul_lo_and_xor(x9, k, vld1q_u64(buf.add(144) as *const u64)); + let y9 = clmul_lo_and_xor(x9, k, load_u64x2(buf.add(144))); x9 = clmul_hi_and_xor(x9, k, y9); - let y10 = clmul_lo_and_xor(x10, k, vld1q_u64(buf.add(160) as *const u64)); + let y10 = clmul_lo_and_xor(x10, k, load_u64x2(buf.add(160))); x10 = clmul_hi_and_xor(x10, k, y10); - let y11 = clmul_lo_and_xor(x11, k, vld1q_u64(buf.add(176) as *const u64)); + let y11 = clmul_lo_and_xor(x11, k, load_u64x2(buf.add(176))); x11 = clmul_hi_and_xor(x11, k, y11); buf = buf.add(192); } @@ -894,11 +993,11 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le crc0 = __crc32d(0, vgetq_lane_u64(x0, 0)); crc0 = __crc32d(crc0, vgetq_lane_u64(x0, 1)); - len = end.offset_from(buf) as usize; + len = end.offset_from_unsigned(buf); } if len >= 16 { - let mut x0 = vld1q_u64(buf as *const u64); + let mut x0 = load_u64x2(buf); let k_vals: [u64; 2] = [0xae689191, 0xccaa009e]; let k = vld1q_u64(k_vals.as_ptr()); @@ -909,7 +1008,7 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le len = len.strict_sub(16); while len >= 16 { - let y0 = clmul_lo_and_xor(x0, k, vld1q_u64(buf as *const u64)); + let y0 = clmul_lo_and_xor(x0, k, load_u64x2(buf)); x0 = clmul_hi_and_xor(x0, k, y0); buf = buf.add(16); len = len.strict_sub(16); @@ -920,7 +1019,7 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le } while len >= 8 { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -937,19 +1036,30 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le /// Safe wrapper for CRC-32 fusion kernel (CRC+PMULL v12e_v1). #[inline] -pub fn crc32_iso_hdlc_pmull_v12e_v1_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_v12e_v1_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC + PMULL before selecting this kernel. unsafe { crc32_iso_hdlc_pmull_v12e_v1(crc, data.as_ptr(), data.len()) } } // Fusion: PMULL v9s3x2e_s3 (no EOR3) +/// XOR three vector-register values. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 NEON target feature is available. #[inline] unsafe fn xor3_u64x2(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { - // SAFETY: Caller guarantees NEON is available. veorq_u64 operates on registers. + // SAFETY: The caller establishes NEON support; both intrinsics operate only on registers. unsafe { veorq_u64(veorq_u64(a, b), c) } } +/// Update CRC-32C with the three-stream PMULL fusion kernel. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. When `len` +/// is nonzero, `buf` must be valid to read `len` initialized bytes from one allocation. #[inline] #[target_feature(enable = "crc,aes")] unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -966,7 +1076,7 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l } if (buf as usize & 8) != 0 && len >= 8 { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -980,15 +1090,15 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l let mut crc1 = 0u32; let mut crc2 = 0u32; - let mut x0 = vld1q_u64(buf2 as *const u64); - let mut x1 = vld1q_u64(buf2.add(16) as *const u64); - let mut x2 = vld1q_u64(buf2.add(32) as *const u64); - let mut x3 = vld1q_u64(buf2.add(48) as *const u64); - let mut x4 = vld1q_u64(buf2.add(64) as *const u64); - let mut x5 = vld1q_u64(buf2.add(80) as *const u64); - let mut x6 = vld1q_u64(buf2.add(96) as *const u64); - let mut x7 = vld1q_u64(buf2.add(112) as *const u64); - let mut x8 = vld1q_u64(buf2.add(128) as *const u64); + let mut x0 = load_u64x2(buf2); + let mut x1 = load_u64x2(buf2.add(16)); + let mut x2 = load_u64x2(buf2.add(32)); + let mut x3 = load_u64x2(buf2.add(48)); + let mut x4 = load_u64x2(buf2.add(64)); + let mut x5 = load_u64x2(buf2.add(80)); + let mut x6 = load_u64x2(buf2.add(96)); + let mut x7 = load_u64x2(buf2.add(112)); + let mut x8 = load_u64x2(buf2.add(128)); let k_vals: [u64; 2] = [0x7e908048, 0xc96cfdc0]; let mut k = vld1q_u64(k_vals.as_ptr()); @@ -1014,25 +1124,22 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l let y8 = clmul_lo(x8, k); x8 = clmul_hi(x8, k); - x0 = xor3_u64x2(x0, y0, vld1q_u64(buf2 as *const u64)); - x1 = xor3_u64x2(x1, y1, vld1q_u64(buf2.add(16) as *const u64)); - x2 = xor3_u64x2(x2, y2, vld1q_u64(buf2.add(32) as *const u64)); - x3 = xor3_u64x2(x3, y3, vld1q_u64(buf2.add(48) as *const u64)); - x4 = xor3_u64x2(x4, y4, vld1q_u64(buf2.add(64) as *const u64)); - x5 = xor3_u64x2(x5, y5, vld1q_u64(buf2.add(80) as *const u64)); - x6 = xor3_u64x2(x6, y6, vld1q_u64(buf2.add(96) as *const u64)); - x7 = xor3_u64x2(x7, y7, vld1q_u64(buf2.add(112) as *const u64)); - x8 = xor3_u64x2(x8, y8, vld1q_u64(buf2.add(128) as *const u64)); - - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen.strict_add(8)) as *const u64)); - crc2 = __crc32cd( - crc2, - ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(8)) as *const u64), - ); + x0 = xor3_u64x2(x0, y0, load_u64x2(buf2)); + x1 = xor3_u64x2(x1, y1, load_u64x2(buf2.add(16))); + x2 = xor3_u64x2(x2, y2, load_u64x2(buf2.add(32))); + x3 = xor3_u64x2(x3, y3, load_u64x2(buf2.add(48))); + x4 = xor3_u64x2(x4, y4, load_u64x2(buf2.add(64))); + x5 = xor3_u64x2(x5, y5, load_u64x2(buf2.add(80))); + x6 = xor3_u64x2(x6, y6, load_u64x2(buf2.add(96))); + x7 = xor3_u64x2(x7, y7, load_u64x2(buf2.add(112))); + x8 = xor3_u64x2(x8, y8, load_u64x2(buf2.add(128))); + + crc0 = __crc32cd(crc0, load_u64(buf)); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32cd(crc0, load_u64(buf.add(8))); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); buf = buf.add(16); buf2 = buf2.add(144); @@ -1084,15 +1191,12 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l x0 = clmul_hi(x0, k); x0 = xor3_u64x2(x0, y0, x4); - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen.strict_add(8)) as *const u64)); - crc2 = __crc32cd( - crc2, - ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(8)) as *const u64), - ); + crc0 = __crc32cd(crc0, load_u64(buf)); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32cd(crc0, load_u64(buf.add(8))); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); let vc0 = crc_shift_iscsi(crc0, klen.strict_mul(2).strict_add(blk.strict_mul(144))); let vc1 = crc_shift_iscsi(crc1, klen.strict_add(blk.strict_mul(144))); @@ -1103,7 +1207,7 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l crc0 = __crc32cd(crc0, vc ^ vgetq_lane_u64(x0, 1)); buf = buf2; - len = end.offset_from(buf) as usize; + len = end.offset_from_unsigned(buf); } if len >= 32 { @@ -1112,9 +1216,9 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l let mut crc2 = 0u32; loop { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2)))); buf = buf.add(8); len = len.strict_sub(24); if len < 32 { @@ -1128,13 +1232,13 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l buf = buf.add(klen.strict_mul(2)); crc0 = crc2; - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64) ^ vc); + crc0 = __crc32cd(crc0, load_u64(buf) ^ vc); buf = buf.add(8); len = len.strict_sub(8); } while len >= 8 { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1151,11 +1255,17 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l /// Safe wrapper for CRC-32C fusion kernel (CRC+PMULL, no EOR3). #[inline] -pub fn crc32c_iscsi_pmull_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC + PMULL before selecting this kernel. unsafe { crc32c_iscsi_pmull_v9s3x2e_s3(crc, data.as_ptr(), data.len()) } } +/// Update CRC-32 (IEEE) with the three-stream PMULL fusion kernel. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. When `len` +/// is nonzero, `buf` must be valid to read `len` initialized bytes from one allocation. #[inline] #[target_feature(enable = "crc,aes")] unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -1172,7 +1282,7 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut } if (buf as usize & 8) != 0 && len >= 8 { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1186,15 +1296,15 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut let mut crc1 = 0u32; let mut crc2 = 0u32; - let mut x0 = vld1q_u64(buf2 as *const u64); - let mut x1 = vld1q_u64(buf2.add(16) as *const u64); - let mut x2 = vld1q_u64(buf2.add(32) as *const u64); - let mut x3 = vld1q_u64(buf2.add(48) as *const u64); - let mut x4 = vld1q_u64(buf2.add(64) as *const u64); - let mut x5 = vld1q_u64(buf2.add(80) as *const u64); - let mut x6 = vld1q_u64(buf2.add(96) as *const u64); - let mut x7 = vld1q_u64(buf2.add(112) as *const u64); - let mut x8 = vld1q_u64(buf2.add(128) as *const u64); + let mut x0 = load_u64x2(buf2); + let mut x1 = load_u64x2(buf2.add(16)); + let mut x2 = load_u64x2(buf2.add(32)); + let mut x3 = load_u64x2(buf2.add(48)); + let mut x4 = load_u64x2(buf2.add(64)); + let mut x5 = load_u64x2(buf2.add(80)); + let mut x6 = load_u64x2(buf2.add(96)); + let mut x7 = load_u64x2(buf2.add(112)); + let mut x8 = load_u64x2(buf2.add(128)); let k_vals: [u64; 2] = [0x26b70c3d, 0x3f41287a]; let mut k = vld1q_u64(k_vals.as_ptr()); @@ -1220,25 +1330,22 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut let y8 = clmul_lo(x8, k); x8 = clmul_hi(x8, k); - x0 = xor3_u64x2(x0, y0, vld1q_u64(buf2 as *const u64)); - x1 = xor3_u64x2(x1, y1, vld1q_u64(buf2.add(16) as *const u64)); - x2 = xor3_u64x2(x2, y2, vld1q_u64(buf2.add(32) as *const u64)); - x3 = xor3_u64x2(x3, y3, vld1q_u64(buf2.add(48) as *const u64)); - x4 = xor3_u64x2(x4, y4, vld1q_u64(buf2.add(64) as *const u64)); - x5 = xor3_u64x2(x5, y5, vld1q_u64(buf2.add(80) as *const u64)); - x6 = xor3_u64x2(x6, y6, vld1q_u64(buf2.add(96) as *const u64)); - x7 = xor3_u64x2(x7, y7, vld1q_u64(buf2.add(112) as *const u64)); - x8 = xor3_u64x2(x8, y8, vld1q_u64(buf2.add(128) as *const u64)); - - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32d(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen.strict_add(8)) as *const u64)); - crc2 = __crc32d( - crc2, - ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(8)) as *const u64), - ); + x0 = xor3_u64x2(x0, y0, load_u64x2(buf2)); + x1 = xor3_u64x2(x1, y1, load_u64x2(buf2.add(16))); + x2 = xor3_u64x2(x2, y2, load_u64x2(buf2.add(32))); + x3 = xor3_u64x2(x3, y3, load_u64x2(buf2.add(48))); + x4 = xor3_u64x2(x4, y4, load_u64x2(buf2.add(64))); + x5 = xor3_u64x2(x5, y5, load_u64x2(buf2.add(80))); + x6 = xor3_u64x2(x6, y6, load_u64x2(buf2.add(96))); + x7 = xor3_u64x2(x7, y7, load_u64x2(buf2.add(112))); + x8 = xor3_u64x2(x8, y8, load_u64x2(buf2.add(128))); + + crc0 = __crc32d(crc0, load_u64(buf)); + crc1 = __crc32d(crc1, load_u64(buf.add(klen))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32d(crc0, load_u64(buf.add(8))); + crc1 = __crc32d(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); buf = buf.add(16); buf2 = buf2.add(144); @@ -1290,15 +1397,12 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut x0 = clmul_hi(x0, k); x0 = xor3_u64x2(x0, y0, x4); - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32d(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen.strict_add(8)) as *const u64)); - crc2 = __crc32d( - crc2, - ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(8)) as *const u64), - ); + crc0 = __crc32d(crc0, load_u64(buf)); + crc1 = __crc32d(crc1, load_u64(buf.add(klen))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32d(crc0, load_u64(buf.add(8))); + crc1 = __crc32d(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); let vc0 = crc_shift_iso_hdlc(crc0, klen.strict_mul(2).strict_add(blk.strict_mul(144))); let vc1 = crc_shift_iso_hdlc(crc1, klen.strict_add(blk.strict_mul(144))); @@ -1309,7 +1413,7 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut crc0 = __crc32d(crc0, vc ^ vgetq_lane_u64(x0, 1)); buf = buf2; - len = end.offset_from(buf) as usize; + len = end.offset_from_unsigned(buf); } if len >= 32 { @@ -1318,9 +1422,9 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut let mut crc2 = 0u32; loop { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); + crc1 = __crc32d(crc1, load_u64(buf.add(klen))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2)))); buf = buf.add(8); len = len.strict_sub(24); if len < 32 { @@ -1334,13 +1438,13 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut buf = buf.add(klen.strict_mul(2)); crc0 = crc2; - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64) ^ vc); + crc0 = __crc32d(crc0, load_u64(buf) ^ vc); buf = buf.add(8); len = len.strict_sub(8); } while len >= 8 { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1357,13 +1461,19 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut /// Safe wrapper for CRC-32 fusion kernel (CRC+PMULL, no EOR3). #[inline] -pub fn crc32_iso_hdlc_pmull_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC + PMULL before selecting this kernel. unsafe { crc32_iso_hdlc_pmull_v9s3x2e_s3(crc, data.as_ptr(), data.len()) } } // Fusion: EOR3 variants (CRC + PMULL + SHA3) +/// Update CRC-32C with the three-stream PMULL/EOR3 fusion kernel. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC, AES/PMULL, and SHA3/EOR3 target features are available. +/// When `len` is nonzero, `buf` must be valid to read `len` initialized bytes from one allocation. #[inline] #[target_feature(enable = "crc,aes,sha3")] unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -1380,7 +1490,7 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, } if (buf as usize & 8) != 0 && len >= 8 { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1394,15 +1504,15 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, let mut crc1 = 0u32; let mut crc2 = 0u32; - let mut x0 = vld1q_u64(buf2 as *const u64); - let mut x1 = vld1q_u64(buf2.add(16) as *const u64); - let mut x2 = vld1q_u64(buf2.add(32) as *const u64); - let mut x3 = vld1q_u64(buf2.add(48) as *const u64); - let mut x4 = vld1q_u64(buf2.add(64) as *const u64); - let mut x5 = vld1q_u64(buf2.add(80) as *const u64); - let mut x6 = vld1q_u64(buf2.add(96) as *const u64); - let mut x7 = vld1q_u64(buf2.add(112) as *const u64); - let mut x8 = vld1q_u64(buf2.add(128) as *const u64); + let mut x0 = load_u64x2(buf2); + let mut x1 = load_u64x2(buf2.add(16)); + let mut x2 = load_u64x2(buf2.add(32)); + let mut x3 = load_u64x2(buf2.add(48)); + let mut x4 = load_u64x2(buf2.add(64)); + let mut x5 = load_u64x2(buf2.add(80)); + let mut x6 = load_u64x2(buf2.add(96)); + let mut x7 = load_u64x2(buf2.add(112)); + let mut x8 = load_u64x2(buf2.add(128)); let k_vals: [u64; 2] = [0x7e908048, 0xc96cfdc0]; let mut k = vld1q_u64(k_vals.as_ptr()); @@ -1428,22 +1538,22 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, let y8 = clmul_lo(x8, k); x8 = clmul_hi(x8, k); - x0 = veor3q_u64(x0, y0, vld1q_u64(buf2 as *const u64)); - x1 = veor3q_u64(x1, y1, vld1q_u64(buf2.add(16) as *const u64)); - x2 = veor3q_u64(x2, y2, vld1q_u64(buf2.add(32) as *const u64)); - x3 = veor3q_u64(x3, y3, vld1q_u64(buf2.add(48) as *const u64)); - x4 = veor3q_u64(x4, y4, vld1q_u64(buf2.add(64) as *const u64)); - x5 = veor3q_u64(x5, y5, vld1q_u64(buf2.add(80) as *const u64)); - x6 = veor3q_u64(x6, y6, vld1q_u64(buf2.add(96) as *const u64)); - x7 = veor3q_u64(x7, y7, vld1q_u64(buf2.add(112) as *const u64)); - x8 = veor3q_u64(x8, y8, vld1q_u64(buf2.add(128) as *const u64)); - - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen + 8) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2) + 8) as *const u64)); + x0 = veor3q_u64(x0, y0, load_u64x2(buf2)); + x1 = veor3q_u64(x1, y1, load_u64x2(buf2.add(16))); + x2 = veor3q_u64(x2, y2, load_u64x2(buf2.add(32))); + x3 = veor3q_u64(x3, y3, load_u64x2(buf2.add(48))); + x4 = veor3q_u64(x4, y4, load_u64x2(buf2.add(64))); + x5 = veor3q_u64(x5, y5, load_u64x2(buf2.add(80))); + x6 = veor3q_u64(x6, y6, load_u64x2(buf2.add(96))); + x7 = veor3q_u64(x7, y7, load_u64x2(buf2.add(112))); + x8 = veor3q_u64(x8, y8, load_u64x2(buf2.add(128))); + + crc0 = __crc32cd(crc0, load_u64(buf)); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32cd(crc0, load_u64(buf.add(8))); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); buf = buf.add(16); buf2 = buf2.add(144); @@ -1495,15 +1605,15 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, x0 = clmul_hi(x0, k); x0 = veor3q_u64(x0, y0, x4); - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen + 8) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2) + 8) as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32cd(crc0, load_u64(buf.add(8))); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); - let vc0 = crc_shift_iscsi(crc0, klen.strict_mul(2) + blk.strict_mul(144)); - let vc1 = crc_shift_iscsi(crc1, klen + blk.strict_mul(144)); + let vc0 = crc_shift_iscsi(crc0, klen.strict_mul(2).strict_add(blk.strict_mul(144))); + let vc1 = crc_shift_iscsi(crc1, klen.strict_add(blk.strict_mul(144))); let vc2 = crc_shift_iscsi(crc2, blk.strict_mul(144)); let vc = vgetq_lane_u64(veor3q_u64(vc0, vc1, vc2), 0); @@ -1511,18 +1621,18 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, crc0 = __crc32cd(crc0, vc ^ vgetq_lane_u64(x0, 1)); buf = buf2; - len = end.offset_from(buf) as usize; + len = end.offset_from_unsigned(buf); } if len >= 32 { - let klen = ((len.strict_sub(8)) / 24).strict_mul(8); + let klen = len.strict_sub(8).strict_div(24).strict_mul(8); let mut crc1 = 0u32; let mut crc2 = 0u32; loop { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2)))); buf = buf.add(8); len = len.strict_sub(24); if len < 32 { @@ -1530,19 +1640,19 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, } } - let vc0 = crc_shift_iscsi(crc0, klen.strict_mul(2) + 8); - let vc1 = crc_shift_iscsi(crc1, klen + 8); + let vc0 = crc_shift_iscsi(crc0, klen.strict_mul(2).strict_add(8)); + let vc1 = crc_shift_iscsi(crc1, klen.strict_add(8)); let vc = vgetq_lane_u64(veorq_u64(vc0, vc1), 0); buf = buf.add(klen.strict_mul(2)); crc0 = crc2; - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64) ^ vc); + crc0 = __crc32cd(crc0, load_u64(buf) ^ vc); buf = buf.add(8); len = len.strict_sub(8); } while len >= 8 { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1557,24 +1667,32 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, } // unsafe } +/// Return the CRC-32C shift factor for `nbytes`. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. #[inline] #[target_feature(enable = "aes")] unsafe fn crc_shift_iscsi(crc: u32, nbytes: usize) -> uint64x2_t { - // SAFETY: Caller guarantees AES (PMULL) is available. clmul_scalar operates on registers. + // SAFETY: The caller establishes CRC and AES/PMULL support; both callees operate on registers. unsafe { clmul_scalar(crc, xnmodp_crc32_iscsi((nbytes.strict_mul(8).strict_sub(33)) as u64)) } } +/// Compute `x^n mod p` for the reflected CRC-32C polynomial. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. #[inline] #[target_feature(enable = "crc,aes")] unsafe fn xnmodp_crc32_iscsi(mut n: u64) -> u32 { - // SAFETY: Caller guarantees CRC+AES target features are available (dispatch check). - // All CRC and NEON intrinsics operate on scalar/register values; no memory access. let mut stack = !1u64; let mut acc: u32; let mut low: u32; while n > 191 { - stack = (stack << 1) + (n & 1); + stack = stack.strict_shl(1) | (n & 1); n = (n >> 1).strict_sub(16); } stack = !stack; @@ -1602,11 +1720,17 @@ unsafe fn xnmodp_crc32_iscsi(mut n: u64) -> u32 { /// Safe wrapper for CRC-32C fusion kernel (CRC+PMULL+EOR3 v9s3x2e_s3). #[inline] -pub fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC + PMULL + SHA3/EOR3 before selecting this kernel. unsafe { crc32c_iscsi_pmull_eor3_v9s3x2e_s3(crc, data.as_ptr(), data.len()) } } +/// Update CRC-32 (IEEE) with the three-stream PMULL/EOR3 fusion kernel. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC, AES/PMULL, and SHA3/EOR3 target features are available. +/// When `len` is nonzero, `buf` must be valid to read `len` initialized bytes from one allocation. #[inline] #[target_feature(enable = "crc,aes,sha3")] unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -1623,7 +1747,7 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 } if (buf as usize & 8) != 0 && len >= 8 { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1637,15 +1761,15 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 let mut crc1 = 0u32; let mut crc2 = 0u32; - let mut x0 = vld1q_u64(buf2 as *const u64); - let mut x1 = vld1q_u64(buf2.add(16) as *const u64); - let mut x2 = vld1q_u64(buf2.add(32) as *const u64); - let mut x3 = vld1q_u64(buf2.add(48) as *const u64); - let mut x4 = vld1q_u64(buf2.add(64) as *const u64); - let mut x5 = vld1q_u64(buf2.add(80) as *const u64); - let mut x6 = vld1q_u64(buf2.add(96) as *const u64); - let mut x7 = vld1q_u64(buf2.add(112) as *const u64); - let mut x8 = vld1q_u64(buf2.add(128) as *const u64); + let mut x0 = load_u64x2(buf2); + let mut x1 = load_u64x2(buf2.add(16)); + let mut x2 = load_u64x2(buf2.add(32)); + let mut x3 = load_u64x2(buf2.add(48)); + let mut x4 = load_u64x2(buf2.add(64)); + let mut x5 = load_u64x2(buf2.add(80)); + let mut x6 = load_u64x2(buf2.add(96)); + let mut x7 = load_u64x2(buf2.add(112)); + let mut x8 = load_u64x2(buf2.add(128)); let k_vals: [u64; 2] = [0x26b70c3d, 0x3f41287a]; let mut k = vld1q_u64(k_vals.as_ptr()); @@ -1671,22 +1795,22 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 let y8 = clmul_lo(x8, k); x8 = clmul_hi(x8, k); - x0 = veor3q_u64(x0, y0, vld1q_u64(buf2 as *const u64)); - x1 = veor3q_u64(x1, y1, vld1q_u64(buf2.add(16) as *const u64)); - x2 = veor3q_u64(x2, y2, vld1q_u64(buf2.add(32) as *const u64)); - x3 = veor3q_u64(x3, y3, vld1q_u64(buf2.add(48) as *const u64)); - x4 = veor3q_u64(x4, y4, vld1q_u64(buf2.add(64) as *const u64)); - x5 = veor3q_u64(x5, y5, vld1q_u64(buf2.add(80) as *const u64)); - x6 = veor3q_u64(x6, y6, vld1q_u64(buf2.add(96) as *const u64)); - x7 = veor3q_u64(x7, y7, vld1q_u64(buf2.add(112) as *const u64)); - x8 = veor3q_u64(x8, y8, vld1q_u64(buf2.add(128) as *const u64)); - - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32d(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen + 8) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2) + 8) as *const u64)); + x0 = veor3q_u64(x0, y0, load_u64x2(buf2)); + x1 = veor3q_u64(x1, y1, load_u64x2(buf2.add(16))); + x2 = veor3q_u64(x2, y2, load_u64x2(buf2.add(32))); + x3 = veor3q_u64(x3, y3, load_u64x2(buf2.add(48))); + x4 = veor3q_u64(x4, y4, load_u64x2(buf2.add(64))); + x5 = veor3q_u64(x5, y5, load_u64x2(buf2.add(80))); + x6 = veor3q_u64(x6, y6, load_u64x2(buf2.add(96))); + x7 = veor3q_u64(x7, y7, load_u64x2(buf2.add(112))); + x8 = veor3q_u64(x8, y8, load_u64x2(buf2.add(128))); + + crc0 = __crc32d(crc0, load_u64(buf)); + crc1 = __crc32d(crc1, load_u64(buf.add(klen))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32d(crc0, load_u64(buf.add(8))); + crc1 = __crc32d(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); buf = buf.add(16); buf2 = buf2.add(144); @@ -1738,15 +1862,15 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 x0 = clmul_hi(x0, k); x0 = veor3q_u64(x0, y0, x4); - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32d(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen + 8) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2) + 8) as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); + crc1 = __crc32d(crc1, load_u64(buf.add(klen))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32d(crc0, load_u64(buf.add(8))); + crc1 = __crc32d(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); - let vc0 = crc_shift_iso_hdlc(crc0, klen.strict_mul(2) + blk.strict_mul(144)); - let vc1 = crc_shift_iso_hdlc(crc1, klen + blk.strict_mul(144)); + let vc0 = crc_shift_iso_hdlc(crc0, klen.strict_mul(2).strict_add(blk.strict_mul(144))); + let vc1 = crc_shift_iso_hdlc(crc1, klen.strict_add(blk.strict_mul(144))); let vc2 = crc_shift_iso_hdlc(crc2, blk.strict_mul(144)); let vc = vgetq_lane_u64(veor3q_u64(vc0, vc1, vc2), 0); @@ -1754,18 +1878,18 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 crc0 = __crc32d(crc0, vc ^ vgetq_lane_u64(x0, 1)); buf = buf2; - len = end.offset_from(buf) as usize; + len = end.offset_from_unsigned(buf); } if len >= 32 { - let klen = ((len.strict_sub(8)) / 24).strict_mul(8); + let klen = len.strict_sub(8).strict_div(24).strict_mul(8); let mut crc1 = 0u32; let mut crc2 = 0u32; loop { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); + crc1 = __crc32d(crc1, load_u64(buf.add(klen))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2)))); buf = buf.add(8); len = len.strict_sub(24); if len < 32 { @@ -1773,19 +1897,19 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 } } - let vc0 = crc_shift_iso_hdlc(crc0, klen.strict_mul(2) + 8); - let vc1 = crc_shift_iso_hdlc(crc1, klen + 8); + let vc0 = crc_shift_iso_hdlc(crc0, klen.strict_mul(2).strict_add(8)); + let vc1 = crc_shift_iso_hdlc(crc1, klen.strict_add(8)); let vc = vgetq_lane_u64(veorq_u64(vc0, vc1), 0); buf = buf.add(klen.strict_mul(2)); crc0 = crc2; - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64) ^ vc); + crc0 = __crc32d(crc0, load_u64(buf) ^ vc); buf = buf.add(8); len = len.strict_sub(8); } while len >= 8 { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1800,24 +1924,32 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 } // unsafe } +/// Return the CRC-32 (IEEE) shift factor for `nbytes`. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. #[inline] #[target_feature(enable = "aes")] unsafe fn crc_shift_iso_hdlc(crc: u32, nbytes: usize) -> uint64x2_t { - // SAFETY: Caller guarantees AES (PMULL) is available. clmul_scalar operates on registers. + // SAFETY: The caller establishes CRC and AES/PMULL support; both callees operate on registers. unsafe { clmul_scalar(crc, xnmodp_iso_hdlc((nbytes.strict_mul(8).strict_sub(33)) as u64)) } } +/// Compute `x^n mod p` for the reflected CRC-32 (IEEE) polynomial. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. #[inline] #[target_feature(enable = "crc,aes")] unsafe fn xnmodp_iso_hdlc(mut n: u64) -> u32 { - // SAFETY: Caller guarantees CRC+AES target features are available (dispatch check). - // All CRC and NEON intrinsics operate on scalar/register values; no memory access. let mut stack = !1u64; let mut acc: u32; let mut low: u32; while n > 191 { - stack = (stack << 1) + (n & 1); + stack = stack.strict_shl(1) | (n & 1); n = (n >> 1).strict_sub(16); } stack = !stack; @@ -1845,7 +1977,7 @@ unsafe fn xnmodp_iso_hdlc(mut n: u64) -> u32 { /// Safe wrapper for CRC-32 fusion kernel (CRC+PMULL+EOR3 v9s3x2e_s3). #[inline] -pub fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC + PMULL + SHA3/EOR3 before selecting this kernel. unsafe { crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(crc, data.as_ptr(), data.len()) } } @@ -1866,25 +1998,34 @@ mod tests { fn make_data(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(31).wrapping_add((i >> 8) as u8)) + .map(|i| { + let [low, high, ..] = i.to_le_bytes(); + low.wrapping_mul(31).wrapping_add(high) + }) .collect() } fn assert_crc32_kernel(name: &str, kernel: fn(u32, &[u8]) -> u32, lens: &[usize]) { for &len in lens { - let data = make_data(len); - let expected = super::super::portable::crc32_slice16_ieee(!0, &data) ^ !0; - let got = kernel(!0, &data) ^ !0; - assert_eq!(got, expected, "{name} len={len}"); + for offset in 0usize..16 { + let storage = make_data(len.strict_add(offset)); + let data = &storage[offset..]; + let expected = super::super::portable::crc32_slice16_ieee(!0, data) ^ !0; + let got = kernel(!0, data) ^ !0; + assert_eq!(got, expected, "{name} len={len} offset={offset}"); + } } } fn assert_crc32c_kernel(name: &str, kernel: fn(u32, &[u8]) -> u32, lens: &[usize]) { for &len in lens { - let data = make_data(len); - let expected = super::super::portable::crc32c_slice16(!0, &data) ^ !0; - let got = kernel(!0, &data) ^ !0; - assert_eq!(got, expected, "{name} len={len}"); + for offset in 0usize..16 { + let storage = make_data(len.strict_add(offset)); + let data = &storage[offset..]; + let expected = super::super::portable::crc32c_slice16(!0, data) ^ !0; + let got = kernel(!0, data) ^ !0; + assert_eq!(got, expected, "{name} len={len} offset={offset}"); + } } } diff --git a/src/checksum/crc32/clmul.rs b/src/checksum/crc32/clmul.rs index 6e67c644..23143cf1 100644 --- a/src/checksum/crc32/clmul.rs +++ b/src/checksum/crc32/clmul.rs @@ -35,16 +35,17 @@ const fn reduce128_crc32(hi: u64, lo: u64, poly: u32) -> u32 { let poly_full: u128 = (1u128 << 32) | (poly as u128); let mut val: u128 = ((hi as u128) << 64) | (lo as u128); - let mut bit: i32 = 127; - while bit >= 32 { - let b = bit as u32; + let mut bit = 128u32; + while bit > 32 { + bit = bit.strict_sub(1); + let b = bit; if ((val >> b) & 1) != 0 { val ^= poly_full << b.strict_sub(32); } - bit = bit.strict_sub(1); } - val as u32 + let bytes = val.to_le_bytes(); + u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) } /// Compute `x^n mod (x^32 + poly)` in GF(2), where `poly` is the **normal** CRC polynomial @@ -114,12 +115,12 @@ const fn compute_mu33(poly: u64) -> u64 { let mut i: u32 = 1; while i <= k { let p_i = (poly >> i) & 1; - let q_j = (inv >> (k - i)) & 1; + let q_j = (inv >> k.strict_sub(i)) & 1; s ^= p_i & q_j; i = i.strict_add(1); } - inv |= s << k; + inv |= s.strict_shl(k); k = k.strict_add(1); } @@ -160,7 +161,7 @@ pub(super) struct Crc32ClmulConstants { impl Crc32ClmulConstants { #[must_use] - pub const fn new(reflected_poly: u32) -> Self { + pub(super) const fn new(reflected_poly: u32) -> Self { let poly = reciprocal_poly_crc32(reflected_poly); let mu = compute_mu33(poly); @@ -185,27 +186,30 @@ impl Crc32ClmulConstants { /// Multi-stream folding constants for CRC-32 CLMUL kernels. #[derive(Clone, Copy, Debug)] -#[allow(dead_code)] // Field subsets vary by architecture (x86_64/aarch64/power vs s390x stream widths). pub(super) struct Crc32StreamConstants { pub fold_256b: (u64, u64), pub fold_512b: (u64, u64), + #[cfg(target_arch = "powerpc64")] pub fold_1024b: (u64, u64), pub combine_4way: [(u64, u64); 3], + #[cfg(target_arch = "powerpc64")] pub combine_8way: [(u64, u64); 7], } impl Crc32StreamConstants { #[must_use] - pub const fn new(reflected_poly: u32) -> Self { + pub(super) const fn new(reflected_poly: u32) -> Self { Self { fold_256b: fold16_coeff_for_bytes_crc32(reflected_poly, 256), fold_512b: fold16_coeff_for_bytes_crc32(reflected_poly, 512), + #[cfg(target_arch = "powerpc64")] fold_1024b: fold16_coeff_for_bytes_crc32(reflected_poly, 1024), combine_4way: [ fold16_coeff_for_bytes_crc32(reflected_poly, 384), fold16_coeff_for_bytes_crc32(reflected_poly, 256), fold16_coeff_for_bytes_crc32(reflected_poly, 128), ], + #[cfg(target_arch = "powerpc64")] combine_8way: [ fold16_coeff_for_bytes_crc32(reflected_poly, 896), fold16_coeff_for_bytes_crc32(reflected_poly, 768), diff --git a/src/checksum/crc32/config.rs b/src/checksum/crc32/config.rs index dd6e986f..64008fc8 100644 --- a/src/checksum/crc32/config.rs +++ b/src/checksum/crc32/config.rs @@ -123,8 +123,16 @@ fn parse_force_env() -> Crc32Force { #[inline] #[must_use] -#[allow(unused_variables)] fn clamp_force_to_caps(requested: Crc32Force, caps: Caps) -> Crc32Force { + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" + )))] + let _ = caps; + match requested { Crc32Force::Auto | Crc32Force::Reference | Crc32Force::Portable => requested, Crc32Force::Hwcrc => { @@ -267,7 +275,7 @@ fn config(caps: Caps) -> Crc32Config { /// platform capabilities. #[inline] #[must_use] -pub fn get() -> Crc32Config { +pub(super) fn get() -> Crc32Config { #[cfg(feature = "std")] { use std::sync::OnceLock; diff --git a/src/checksum/crc32/kernels.rs b/src/checksum/crc32/kernels.rs index 12f64d59..c173cae9 100644 --- a/src/checksum/crc32/kernels.rs +++ b/src/checksum/crc32/kernels.rs @@ -14,21 +14,21 @@ //! - Tier 4 (Wide): VPCLMUL (x86_64), PMULL+EOR3/SVE2 (aarch64), Zvbc (riscv64) /// Portable fallback kernel name. -pub use kernels::PORTABLE_SLICE16 as PORTABLE; +pub(in crate::checksum) use kernels::PORTABLE_SLICE16 as PORTABLE; /// Reference (bitwise) kernel name - always available for force mode. -pub use kernels::REFERENCE; +pub(in crate::checksum) use kernels::REFERENCE; use crate::checksum::common::kernels; // Kernel Name Tables (per architecture) #[cfg(target_arch = "x86_64")] -pub mod x86_64 { +pub(in crate::checksum) mod x86_64 { use super::super::x86_64 as arch; use crate::checksum::dispatchers::Crc32Fn; /// CRC-32 (IEEE) PCLMUL kernels. - pub const CRC32_PCLMUL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_PCLMUL: [Crc32Fn; 5] = [ arch::crc32_ieee_pclmul_safe, arch::crc32_ieee_pclmul_2way_safe, arch::crc32_ieee_pclmul_4way_safe, @@ -36,10 +36,10 @@ pub mod x86_64 { arch::crc32_ieee_pclmul_8way_safe, ]; /// CRC-32 (IEEE) PCLMUL small-buffer kernel. - pub const CRC32_PCLMUL_SMALL_KERNEL: Crc32Fn = arch::crc32_ieee_pclmul_small_safe; + pub(in crate::checksum) const CRC32_PCLMUL_SMALL_KERNEL: Crc32Fn = arch::crc32_ieee_pclmul_small_safe; /// CRC-32 (IEEE) VPCLMUL kernels. - pub const CRC32_VPCLMUL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_VPCLMUL: [Crc32Fn; 5] = [ arch::crc32_ieee_vpclmul_safe, arch::crc32_ieee_vpclmul_2way_safe, arch::crc32_ieee_vpclmul_4way_safe, @@ -47,7 +47,7 @@ pub mod x86_64 { arch::crc32_ieee_vpclmul_8way_safe, ]; /// CRC-32C SSE4.2 kernel function array. - pub const CRC32C_HWCRC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_HWCRC: [Crc32Fn; 5] = [ arch::crc32c_sse42_safe, arch::crc32c_sse42_2way_safe, arch::crc32c_sse42_4way_safe, @@ -56,7 +56,7 @@ pub mod x86_64 { ]; /// CRC-32C fusion (SSE4.2 + PCLMULQDQ) kernels. - pub const CRC32C_FUSION_SSE: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_FUSION_SSE: [Crc32Fn; 5] = [ arch::crc32c_iscsi_sse_v4s3x3_safe, arch::crc32c_iscsi_sse_v4s3x3_2way_safe, arch::crc32c_iscsi_sse_v4s3x3_4way_safe, @@ -65,7 +65,7 @@ pub mod x86_64 { ]; /// CRC-32C fusion (AVX-512 + VPCLMULQDQ) kernels. - pub const CRC32C_FUSION_VPCLMUL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_FUSION_VPCLMUL: [Crc32Fn; 5] = [ arch::crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe, arch::crc32c_iscsi_avx512_vpclmulqdq_v3x2_2way_safe, arch::crc32c_iscsi_avx512_vpclmulqdq_v3x2_4way_safe, @@ -75,14 +75,13 @@ pub mod x86_64 { } #[cfg(target_arch = "aarch64")] -pub mod aarch64 { - #![cfg_attr(not(any(target_os = "linux", target_os = "android")), allow(dead_code))] +pub(in crate::checksum) mod aarch64 { use super::super::aarch64 as arch; use crate::checksum::dispatchers::Crc32Fn; /// CRC-32 (IEEE) CRC-extension kernel function array. - pub const CRC32_HWCRC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_HWCRC: [Crc32Fn; 5] = [ arch::crc32_armv8_safe, arch::crc32_armv8_2way_safe, arch::crc32_armv8_3way_safe, @@ -91,7 +90,7 @@ pub mod aarch64 { ]; /// CRC-32C (Castagnoli) CRC-extension kernel function array. - pub const CRC32C_HWCRC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_HWCRC: [Crc32Fn; 5] = [ arch::crc32c_armv8_safe, arch::crc32c_armv8_2way_safe, arch::crc32c_armv8_3way_safe, @@ -100,7 +99,7 @@ pub mod aarch64 { ]; /// CRC-32 (IEEE) PMULL fusion kernels. - pub const CRC32_PMULL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_PMULL: [Crc32Fn; 5] = [ arch::crc32_iso_hdlc_pmull_v9s3x2e_s3_safe, arch::crc32_iso_hdlc_pmull_2way_safe, arch::crc32_iso_hdlc_pmull_3way_safe, @@ -108,10 +107,10 @@ pub mod aarch64 { arch::crc32_iso_hdlc_pmull_3way_safe, // dup for index consistency ]; /// CRC-32 (IEEE) PMULL small-buffer kernel. - pub const CRC32_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32_iso_hdlc_pmull_small_safe; + pub(in crate::checksum) const CRC32_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32_iso_hdlc_pmull_small_safe; /// CRC-32 (IEEE) PMULL+EOR3 fusion kernels. - pub const CRC32_PMULL_EOR3: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_PMULL_EOR3: [Crc32Fn; 5] = [ arch::crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3_safe, arch::crc32_iso_hdlc_pmull_eor3_2way_safe, arch::crc32_iso_hdlc_pmull_eor3_3way_safe, @@ -120,7 +119,7 @@ pub mod aarch64 { ]; /// CRC-32C (Castagnoli) PMULL fusion kernels. - pub const CRC32C_PMULL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_PMULL: [Crc32Fn; 5] = [ arch::crc32c_iscsi_pmull_v9s3x2e_s3_safe, arch::crc32c_iscsi_pmull_2way_safe, arch::crc32c_iscsi_pmull_3way_safe, @@ -128,10 +127,10 @@ pub mod aarch64 { arch::crc32c_iscsi_pmull_3way_safe, // dup for index consistency ]; /// CRC-32C PMULL small-buffer kernel. - pub const CRC32C_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32c_iscsi_pmull_small_safe; + pub(in crate::checksum) const CRC32C_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32c_iscsi_pmull_small_safe; /// CRC-32C (Castagnoli) PMULL+EOR3 fusion kernels. - pub const CRC32C_PMULL_EOR3: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_PMULL_EOR3: [Crc32Fn; 5] = [ arch::crc32c_iscsi_pmull_eor3_v9s3x2e_s3_safe, arch::crc32c_iscsi_pmull_eor3_2way_safe, arch::crc32c_iscsi_pmull_eor3_3way_safe, @@ -139,36 +138,40 @@ pub mod aarch64 { arch::crc32c_iscsi_pmull_eor3_3way_safe, // dup for index consistency ]; + #[cfg(feature = "std")] /// CRC-32 (IEEE) "SVE2 PMULL" tier kernels (2/3-way striping). - pub const CRC32_SVE2_PMULL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_SVE2_PMULL: [Crc32Fn; 5] = [ arch::crc32_iso_hdlc_pmull_v12e_v1_safe, arch::crc32_iso_hdlc_sve2_pmull_2way_safe, arch::crc32_iso_hdlc_sve2_pmull_3way_safe, arch::crc32_iso_hdlc_sve2_pmull_3way_safe, arch::crc32_iso_hdlc_sve2_pmull_3way_safe, ]; + #[cfg(feature = "std")] /// CRC-32 "SVE2 PMULL" small-buffer kernel. - pub const CRC32_SVE2_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32_iso_hdlc_sve2_pmull_small_safe; + pub(in crate::checksum) const CRC32_SVE2_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32_iso_hdlc_sve2_pmull_small_safe; + #[cfg(feature = "std")] /// CRC-32C (Castagnoli) "SVE2 PMULL" tier kernels (2/3-way striping). - pub const CRC32C_SVE2_PMULL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_SVE2_PMULL: [Crc32Fn; 5] = [ arch::crc32c_iscsi_pmull_v12e_v1_safe, arch::crc32c_iscsi_sve2_pmull_2way_safe, arch::crc32c_iscsi_sve2_pmull_3way_safe, arch::crc32c_iscsi_sve2_pmull_3way_safe, arch::crc32c_iscsi_sve2_pmull_3way_safe, ]; + #[cfg(feature = "std")] /// CRC-32C "SVE2 PMULL" small-buffer kernel. - pub const CRC32C_SVE2_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32c_iscsi_sve2_pmull_small_safe; + pub(in crate::checksum) const CRC32C_SVE2_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32c_iscsi_sve2_pmull_small_safe; } #[cfg(target_arch = "powerpc64")] -pub mod power { +pub(in crate::checksum) mod power { use super::super::power as arch; use crate::checksum::dispatchers::Crc32Fn; /// CRC-32 (IEEE) VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)]. - pub const CRC32_VPMSUM: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_VPMSUM: [Crc32Fn; 5] = [ arch::crc32_ieee_vpmsum_safe, arch::crc32_ieee_vpmsum_2way_safe, arch::crc32_ieee_vpmsum_4way_safe, @@ -177,7 +180,7 @@ pub mod power { ]; /// CRC-32C VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)]. - pub const CRC32C_VPMSUM: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_VPMSUM: [Crc32Fn; 5] = [ arch::crc32c_vpmsum_safe, arch::crc32c_vpmsum_2way_safe, arch::crc32c_vpmsum_4way_safe, @@ -187,12 +190,12 @@ pub mod power { } #[cfg(target_arch = "s390x")] -pub mod s390x { +pub(in crate::checksum) mod s390x { use super::super::s390x as arch; use crate::checksum::dispatchers::Crc32Fn; /// CRC-32 (IEEE) VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32_VGFM: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_VGFM: [Crc32Fn; 5] = [ arch::crc32_ieee_vgfm_safe, arch::crc32_ieee_vgfm_2way_safe, arch::crc32_ieee_vgfm_4way_safe, @@ -201,7 +204,7 @@ pub mod s390x { ]; /// CRC-32C VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32C_VGFM: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_VGFM: [Crc32Fn; 5] = [ arch::crc32c_vgfm_safe, arch::crc32c_vgfm_2way_safe, arch::crc32c_vgfm_4way_safe, @@ -211,31 +214,12 @@ pub mod s390x { } #[cfg(target_arch = "riscv64")] -#[allow(dead_code)] -pub mod riscv64 { +pub(in crate::checksum) mod riscv64 { use super::super::riscv64 as arch; use crate::checksum::dispatchers::Crc32Fn; - /// Zbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32_ZBC_NAMES: &[&str] = &[ - "riscv64/zbc", - "riscv64/zbc-2way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", - ]; - - /// Zvbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32_ZVBC_NAMES: &[&str] = &[ - "riscv64/zvbc", - "riscv64/zvbc-2way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - ]; - /// CRC-32 (IEEE) Zbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32_ZBC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_ZBC: [Crc32Fn; 5] = [ arch::crc32_ieee_zbc_safe, arch::crc32_ieee_zbc_2way_safe, arch::crc32_ieee_zbc_4way_safe, @@ -244,7 +228,7 @@ pub mod riscv64 { ]; /// CRC-32 (IEEE) Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32_ZVBC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_ZVBC: [Crc32Fn; 5] = [ arch::crc32_ieee_zvbc_safe, arch::crc32_ieee_zvbc_2way_safe, arch::crc32_ieee_zvbc_4way_safe, @@ -252,26 +236,8 @@ pub mod riscv64 { arch::crc32_ieee_zvbc_4way_safe, // dup for index consistency ]; - /// CRC-32C Zbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32C_ZBC_NAMES: &[&str] = &[ - "riscv64/zbc", - "riscv64/zbc-2way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", - ]; - - /// CRC-32C Zvbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32C_ZVBC_NAMES: &[&str] = &[ - "riscv64/zvbc", - "riscv64/zvbc-2way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - ]; - /// CRC-32C Zbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32C_ZBC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_ZBC: [Crc32Fn; 5] = [ arch::crc32c_zbc_safe, arch::crc32c_zbc_2way_safe, arch::crc32c_zbc_4way_safe, @@ -280,7 +246,7 @@ pub mod riscv64 { ]; /// CRC-32C Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32C_ZVBC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_ZVBC: [Crc32Fn; 5] = [ arch::crc32c_zvbc_safe, arch::crc32c_zvbc_2way_safe, arch::crc32c_zvbc_4way_safe, diff --git a/src/checksum/crc32/mod.rs b/src/checksum/crc32/mod.rs index 9a1a0ca3..5ba8b47d 100644 --- a/src/checksum/crc32/mod.rs +++ b/src/checksum/crc32/mod.rs @@ -14,29 +14,6 @@ //! - riscv64: ZVBC (RVV vector CLMUL) / Zbc folding (CRC-32 and CRC-32C) //! - wasm32/wasm64: portable only (no CRC32/CLMUL instructions) //! -//! # Quick Start -//! -//! ```rust -//! use rscrypto::checksum::{Checksum, ChecksumCombine, Crc32, Crc32C}; -//! -//! let data = b"123456789"; -//! -//! // One-shot -//! assert_eq!(Crc32::checksum(data), 0xCBF4_3926); -//! assert_eq!(Crc32C::checksum(data), 0xE306_9283); -//! -//! // Streaming -//! let mut hasher = Crc32::new(); -//! hasher.update(b"1234"); -//! hasher.update(b"56789"); -//! assert_eq!(hasher.finalize(), Crc32::checksum(data)); -//! -//! // Combine: crc(A || B) == combine(crc(A), crc(B), len(B)) -//! let (a, b) = data.split_at(4); -//! let combined = Crc32::combine(Crc32::checksum(a), Crc32::checksum(b), b.len()); -//! assert_eq!(combined, Crc32::checksum(data)); -//! ``` - #[cfg(any(target_arch = "powerpc64", target_arch = "s390x", target_arch = "riscv64"))] mod clmul; pub(crate) mod config; @@ -58,7 +35,6 @@ mod s390x; #[cfg(target_arch = "riscv64")] mod riscv64; -#[allow(unused_imports)] pub use config::{Crc32Config, Crc32Force}; #[cfg(any(test, feature = "std"))] @@ -69,16 +45,14 @@ use crate::checksum::common::{ }; #[cfg(feature = "diag")] use crate::checksum::diag::{Crc32Polynomial, Crc32SelectionDiag}; -#[allow(unused_imports)] -pub(super) use crate::traits::{Checksum, ChecksumCombine}; // Kernel Tables (compile-time) /// Portable kernel tables (pre-computed at compile time). mod kernel_tables { use super::*; - pub static IEEE_TABLES_16: [[u32; 256]; 16] = generate_crc32_tables_16(CRC32_IEEE_POLY); - pub static CRC32C_TABLES_16: [[u32; 256]; 16] = generate_crc32_tables_16(CRC32C_POLY); + pub(super) static IEEE_TABLES_16: [[u32; 256]; 16] = generate_crc32_tables_16(CRC32_IEEE_POLY); + pub(super) static CRC32C_TABLES_16: [[u32; 256]; 16] = generate_crc32_tables_16(CRC32C_POLY); } /// Block size for CRC-32 folding operations. @@ -87,8 +61,7 @@ pub(crate) const CRC32_FOLD_BLOCK_BYTES: usize = 128; // Portable Kernel Wrappers -#[cfg(any(test, feature = "std"))] -#[cfg_attr(all(test, not(feature = "std")), allow(dead_code))] +#[cfg(feature = "std")] fn crc32_portable(crc: u32, data: &[u8]) -> u32 { const THRESHOLD: usize = 64; if data.len() < THRESHOLD { @@ -98,8 +71,7 @@ fn crc32_portable(crc: u32, data: &[u8]) -> u32 { } } -#[cfg(any(test, feature = "std"))] -#[cfg_attr(all(test, not(feature = "std")), allow(dead_code))] +#[cfg(feature = "std")] fn crc32c_portable(crc: u32, data: &[u8]) -> u32 { const THRESHOLD: usize = 64; if data.len() < THRESHOLD { @@ -791,6 +763,7 @@ impl crate::traits::ChecksumCombine for Crc32 { #[cfg(feature = "alloc")] impl Crc32 { + /// Creates a buffering wrapper that coalesces short updates before CRC-32/IEEE dispatch. #[must_use] pub fn buffered() -> BufferedCrc32 { BufferedCrc32::new() @@ -934,6 +907,7 @@ impl crate::traits::ChecksumCombine for Crc32C { #[cfg(feature = "alloc")] impl Crc32C { + /// Creates a buffering wrapper that coalesces short updates before CRC-32C dispatch. #[must_use] pub fn buffered() -> BufferedCrc32C { BufferedCrc32C::new() @@ -991,9 +965,18 @@ mod tests { use alloc::{string::String, vec::Vec}; use super::*; + use crate::traits::{Checksum, ChecksumCombine}; const TEST_DATA: &[u8] = b"123456789"; + fn patterned_data(len: usize, multiplier: u8, addend: u8) -> Vec { + (0u8..=u8::MAX) + .cycle() + .take(len) + .map(|byte| byte.wrapping_mul(multiplier).wrapping_add(addend)) + .collect() + } + #[test] fn test_crc32_test_vectors() { assert_eq!(Crc32::checksum(TEST_DATA), 0xCBF4_3926); @@ -1063,10 +1046,7 @@ mod tests { #[test] fn test_crc32_various_lengths_streaming_matches_oneshot() { - let mut data = [0u8; 512]; - for (i, b) in data.iter_mut().enumerate() { - *b = (i as u8).wrapping_mul(17).wrapping_add(i as u8); - } + let data = patterned_data(512, 18, 0); for &len in &[0usize, 1, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 255, 256, 400, 512] { let slice = &data[..len]; @@ -1098,7 +1078,7 @@ mod tests { #[cfg(feature = "alloc")] #[test] fn test_buffered_crc32_matches_unbuffered() { - let data: Vec = (0..2048).map(|i| (i as u8).wrapping_mul(31)).collect(); + let data = patterned_data(2048, 31, 0); let expected = Crc32::checksum(&data); let mut buffered = BufferedCrc32::new(); @@ -1111,7 +1091,7 @@ mod tests { #[cfg(feature = "alloc")] #[test] fn test_buffered_crc32c_matches_unbuffered() { - let data: Vec = (0..2048).map(|i| (i as u8).wrapping_mul(29).wrapping_add(7)).collect(); + let data = patterned_data(2048, 29, 7); let expected = Crc32C::checksum(&data); let mut buffered = BufferedCrc32C::new(); @@ -1135,7 +1115,7 @@ mod tests { for &threshold in &crc32_thresholds { let size = threshold + 256; - let data: Vec = (0..size).map(|i| (i as u8).wrapping_mul(13)).collect(); + let data = patterned_data(size, 13, 0); let oneshot32 = Crc32::checksum(&data); @@ -1148,7 +1128,7 @@ mod tests { // Same thresholds for CRC32C for &threshold in &crc32_thresholds { let size = threshold + 256; - let data: Vec = (0..size).map(|i| (i as u8).wrapping_mul(13)).collect(); + let data = patterned_data(size, 13, 0); let oneshot32c = Crc32C::checksum(&data); @@ -1173,7 +1153,7 @@ mod tests { } let len = 64 * 1024; - let data: Vec = (0..len).map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)).collect(); + let data = patterned_data(len, 31, 7); let expected = portable::crc32_slice16_ieee(!0, &data) ^ !0; let got = Crc32::checksum(&data); assert_eq!(got, expected); @@ -1234,7 +1214,7 @@ mod tests { } let len = 64 * 1024; - let data: Vec = (0..len).map(|i| (i as u8).wrapping_mul(29).wrapping_add(7)).collect(); + let data = patterned_data(len, 29, 7); let expected = portable::crc32c_slice16(!0, &data) ^ !0; let got = Crc32C::checksum(&data); assert_eq!(got, expected); diff --git a/src/checksum/crc32/portable.rs b/src/checksum/crc32/portable.rs index 20cada34..be43345e 100644 --- a/src/checksum/crc32/portable.rs +++ b/src/checksum/crc32/portable.rs @@ -16,7 +16,7 @@ pub(crate) const BYTEWISE_KERNEL_NAME: &str = "portable/bytewise"; /// /// Uses one 256-entry table rather than the slice-by-16 table set. #[inline(always)] -pub fn crc32_bytewise_ieee(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32_bytewise_ieee(crc: u32, data: &[u8]) -> u32 { crc32_bytewise(crc, data, &kernel_tables::IEEE_TABLES_16[0]) } @@ -24,19 +24,19 @@ pub fn crc32_bytewise_ieee(crc: u32, data: &[u8]) -> u32 { /// /// Uses one 256-entry table rather than the slice-by-16 table set. #[inline(always)] -pub fn crc32c_bytewise(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32c_bytewise(crc: u32, data: &[u8]) -> u32 { crc32_bytewise(crc, data, &kernel_tables::CRC32C_TABLES_16[0]) } /// CRC-32 (IEEE) slice-by-16 computation. #[inline] -pub fn crc32_slice16_ieee(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32_slice16_ieee(crc: u32, data: &[u8]) -> u32 { crc32_slice16(crc, data, &kernel_tables::IEEE_TABLES_16) } /// CRC-32C (Castagnoli) slice-by-16 computation. #[inline] -pub fn crc32c_slice16(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32c_slice16(crc: u32, data: &[u8]) -> u32 { crc32_slice16(crc, data, &kernel_tables::CRC32C_TABLES_16) } @@ -44,8 +44,7 @@ pub fn crc32c_slice16(crc: u32, data: &[u8]) -> u32 { /// Update CRC-32 state using a byte-at-a-time lookup table. #[inline(always)] -#[allow(clippy::indexing_slicing)] // index is 0..=255 by mask, table is [u32; 256] -pub fn crc32_bytewise(mut crc: u32, data: &[u8], table: &[u32; 256]) -> u32 { +fn crc32_bytewise(mut crc: u32, data: &[u8], table: &[u32; 256]) -> u32 { for &b in data { let index = ((crc ^ (b as u32)) & 0xFF) as usize; crc = table[index] ^ (crc >> 8); @@ -55,6 +54,6 @@ pub fn crc32_bytewise(mut crc: u32, data: &[u8], table: &[u32; 256]) -> u32 { /// Update CRC-32 state using slice-by-16 algorithm. #[inline] -pub fn crc32_slice16(crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { +fn crc32_slice16(crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { portable::slice16_32(crc, data, tables) } diff --git a/src/checksum/crc32/power.rs b/src/checksum/crc32/power.rs index f827b352..8000cfd2 100644 --- a/src/checksum/crc32/power.rs +++ b/src/checksum/crc32/power.rs @@ -8,10 +8,7 @@ //! Uses `unsafe` for Power SIMD + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher // SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, @@ -19,6 +16,8 @@ use core::{ simd::i64x2, }; +use crate::checksum::common::low_u32; + use super::clmul::{Crc32ClmulConstants, Crc32StreamConstants}; #[repr(transparent)] @@ -78,6 +77,10 @@ impl Simd { /// On `powerpc64le` this is a no-op. On big-endian `powerpc64`, we byte-swap /// each 64-bit lane so the folding algorithm sees the same lane values as on /// little-endian platforms. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn to_le(self) -> Self { @@ -103,6 +106,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -125,6 +131,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -144,6 +153,10 @@ impl Simd { /// Fold 16 bytes (reflected CRC32 folding primitive): /// `self.low ⊗ coeff.high ⊕ self.high ⊗ coeff.low`. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -162,6 +175,10 @@ impl Simd { } /// Fold 16B → CRC32 width (reflected), returning an intermediate 128-bit state. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -190,6 +207,10 @@ impl Simd { } /// Barrett reduction for reflected CRC32; returns the updated (pre-inverted) CRC. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -204,13 +225,16 @@ impl Simd { // Mirror the x86 reduction scheme (2 multiplies + xor, extract hi32). let t1 = Self::mul64(self.low_64(), mu); let l = Self::mul64(t1.low_64(), poly); - (self ^ l).high_64() as u32 + low_u32((self ^ l).high_64()) } } } // Folding helpers +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline(always)] unsafe fn fold_tail(x: [Simd; 8], consts: &Crc32ClmulConstants) -> u32 { // SAFETY: All target-feature-gated calls require power8-crypto, ensured by callers' @@ -239,6 +263,9 @@ unsafe fn fold_tail(x: [Simd; 8], consts: &Crc32ClmulConstants) -> u32 { } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -261,6 +288,9 @@ unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -293,6 +323,9 @@ unsafe fn update_simd(state: u32, first: &[Simd; 8], rest: &[[Simd; 8]], consts: } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -348,7 +381,7 @@ unsafe fn update_simd_2way( let mut i = 2usize; while i < even { fold_block_128(&mut s0, &blocks[i], coeff_256); - fold_block_128(&mut s1, &blocks[i + 1], coeff_256); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_256); i = i.strict_add(2); } @@ -371,6 +404,9 @@ unsafe fn update_simd_2way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -396,7 +432,7 @@ unsafe fn update_simd_4way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 4) * 4; + let aligned = blocks.len() & !3_usize; let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); @@ -450,9 +486,9 @@ unsafe fn update_simd_4way( let mut i = 4usize; while i < aligned { fold_block_128(&mut s0, &blocks[i], coeff_512); - fold_block_128(&mut s1, &blocks[i + 1], coeff_512); - fold_block_128(&mut s2, &blocks[i + 2], coeff_512); - fold_block_128(&mut s3, &blocks[i + 3], coeff_512); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_512); i = i.strict_add(4); } @@ -503,6 +539,9 @@ unsafe fn update_simd_4way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -528,7 +567,7 @@ unsafe fn update_simd_8way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len() & !7_usize; let coeff_1024 = Simd::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); @@ -568,13 +607,13 @@ unsafe fn update_simd_8way( let mut idx = 8usize; while idx < aligned { fold_block_128(&mut s0, &blocks[idx], coeff_1024); - fold_block_128(&mut s1, &blocks[idx + 1], coeff_1024); - fold_block_128(&mut s2, &blocks[idx + 2], coeff_1024); - fold_block_128(&mut s3, &blocks[idx + 3], coeff_1024); - fold_block_128(&mut s4, &blocks[idx + 4], coeff_1024); - fold_block_128(&mut s5, &blocks[idx + 5], coeff_1024); - fold_block_128(&mut s6, &blocks[idx + 6], coeff_1024); - fold_block_128(&mut s7, &blocks[idx + 7], coeff_1024); + fold_block_128(&mut s1, &blocks[idx.strict_add(1)], coeff_1024); + fold_block_128(&mut s2, &blocks[idx.strict_add(2)], coeff_1024); + fold_block_128(&mut s3, &blocks[idx.strict_add(3)], coeff_1024); + fold_block_128(&mut s4, &blocks[idx.strict_add(4)], coeff_1024); + fold_block_128(&mut s5, &blocks[idx.strict_add(5)], coeff_1024); + fold_block_128(&mut s6, &blocks[idx.strict_add(6)], coeff_1024); + fold_block_128(&mut s7, &blocks[idx.strict_add(7)], coeff_1024); idx = idx.strict_add(8); } @@ -611,6 +650,9 @@ unsafe fn update_simd_8way( // Public kernels (IEEE + CRC32C) +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -635,6 +677,9 @@ unsafe fn crc32_fold_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -658,6 +703,9 @@ unsafe fn crc32c_fold_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -693,6 +741,9 @@ unsafe fn crc32_fold_kernel_nway( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -731,13 +782,13 @@ unsafe fn crc32c_fold_kernel_nway( // Safe wrappers (dispatcher entrypoints) #[inline] -pub fn crc32_ieee_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32_fold_kernel(crc, data, &super::clmul::CRC32_IEEE_CLMUL) } } #[inline] -pub fn crc32_ieee_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32_fold_kernel_nway::<2>( @@ -750,7 +801,7 @@ pub fn crc32_ieee_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32_fold_kernel_nway::<4>( @@ -763,7 +814,7 @@ pub fn crc32_ieee_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32_fold_kernel_nway::<8>( @@ -776,25 +827,25 @@ pub fn crc32_ieee_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32c_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32c_fold_kernel(crc, data, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32c_fold_kernel_nway::<2>(crc, data, &super::clmul::CRC32C_STREAM, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32c_fold_kernel_nway::<4>(crc, data, &super::clmul::CRC32C_STREAM, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32c_fold_kernel_nway::<8>(crc, data, &super::clmul::CRC32C_STREAM, &super::clmul::CRC32C_CLMUL) } } diff --git a/src/checksum/crc32/riscv64.rs b/src/checksum/crc32/riscv64.rs index ad20d05b..c8e22de2 100644 --- a/src/checksum/crc32/riscv64.rs +++ b/src/checksum/crc32/riscv64.rs @@ -9,16 +9,14 @@ //! Uses `unsafe` for RISC-V inline assembly. Callers must ensure the required //! CPU features are available before executing the accelerated path (the //! dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, ops::{BitXor, BitXorAssign}, }; +use crate::checksum::common::low_u32; + use super::clmul::{Crc32ClmulConstants, Crc32StreamConstants}; type Block = [u64; 16]; // 128 bytes (8×16B lanes) @@ -68,8 +66,11 @@ impl Simd { // Zbc carryless multiply primitives #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn clmul_lo(a: u64, b: u64) -> u64 { + fn clmul_lo(a: u64, b: u64) -> u64 { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -86,8 +87,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn clmul_hi(a: u64, b: u64) -> u64 { + fn clmul_hi(a: u64, b: u64) -> u64 { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -104,15 +108,14 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - Self { - hi: Self::clmul_hi(a, b), - lo: Self::clmul_lo(a, b), - } + fn mul64(a: u64, b: u64) -> Self { + Self { + hi: Self::clmul_hi(a, b), + lo: Self::clmul_lo(a, b), } } } @@ -122,13 +125,8 @@ impl Simd { #[inline(always)] fn load_block(block: &Block) -> [Simd; 8] { let mut out = [Simd::new(0, 0); 8]; - - let mut i = 0; - while i < 8 { - let low = u64::from_le(block[i * 2]); - let high = u64::from_le(block[i * 2 + 1]); - out[i] = Simd::new(high, low); - i = i.strict_add(1); + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new(u64::from_le(high), u64::from_le(low)); } out @@ -137,256 +135,243 @@ fn load_block(block: &Block) -> [Simd; 8] { // Zbc folding helpers (scalar) #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_16_zbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - // Reflected CRC32 fold primitive is cross-term: low×high ⊕ high×low. - Simd::mul64(x.low_64(), coeff_high) ^ Simd::mul64(x.high_64(), coeff_low) - } +fn fold_16_zbc(x: Simd, coeff: (u64, u64)) -> Simd { + let (coeff_high, coeff_low) = coeff; + // Reflected CRC32 fold primitive is cross-term: low×high ⊕ high×low. + Simd::mul64(x.low_64(), coeff_high) ^ Simd::mul64(x.high_64(), coeff_low) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_width_crc32_reflected_zbc(x: Simd, high: u64, low: u64) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let clmul = Simd::mul64(x.low_64(), low); - let shifted = Simd::new(0, x.high_64()); - let mut state = clmul ^ shifted; +fn fold_width_crc32_reflected_zbc(x: Simd, high: u64, low: u64) -> Simd { + let clmul = Simd::mul64(x.low_64(), low); + let shifted = Simd::new(0, x.high_64()); + let mut state = clmul ^ shifted; - let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = Simd::mul64(shifted_high, high); - state = clmul ^ masked; + let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = Simd::mul64(shifted_high, high); + state = clmul ^ masked; - state - } + state } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn barrett_crc32_reflected_zbc(x: Simd, poly: u64, mu: u64) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = Simd::mul64(x.low_64(), mu); - let l = Simd::mul64(t1.low_64(), poly); - (x ^ l).high_64() as u32 - } +fn barrett_crc32_reflected_zbc(x: Simd, poly: u64, mu: u64) -> u32 { + let t1 = Simd::mul64(x.low_64(), mu); + let l = Simd::mul64(t1.low_64(), poly); + low_u32((x ^ l).high_64()) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_tail_zbc(x: [Simd; 8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut acc = x[7]; - acc ^= fold_16_zbc(x[0], consts.tail_fold_16b[0]); - acc ^= fold_16_zbc(x[1], consts.tail_fold_16b[1]); - acc ^= fold_16_zbc(x[2], consts.tail_fold_16b[2]); - acc ^= fold_16_zbc(x[3], consts.tail_fold_16b[3]); - acc ^= fold_16_zbc(x[4], consts.tail_fold_16b[4]); - acc ^= fold_16_zbc(x[5], consts.tail_fold_16b[5]); - acc ^= fold_16_zbc(x[6], consts.tail_fold_16b[6]); - - let (fold_width_high, fold_width_low) = consts.fold_width; - barrett_crc32_reflected_zbc( - fold_width_crc32_reflected_zbc(acc, fold_width_high, fold_width_low), - consts.poly, - consts.mu, - ) - } +fn fold_tail_zbc(x: [Simd; 8], consts: &Crc32ClmulConstants) -> u32 { + let mut acc = x[7]; + acc ^= fold_16_zbc(x[0], consts.tail_fold_16b[0]); + acc ^= fold_16_zbc(x[1], consts.tail_fold_16b[1]); + acc ^= fold_16_zbc(x[2], consts.tail_fold_16b[2]); + acc ^= fold_16_zbc(x[3], consts.tail_fold_16b[3]); + acc ^= fold_16_zbc(x[4], consts.tail_fold_16b[4]); + acc ^= fold_16_zbc(x[5], consts.tail_fold_16b[5]); + acc ^= fold_16_zbc(x[6], consts.tail_fold_16b[6]); + + let (fold_width_high, fold_width_low) = consts.fold_width; + barrett_crc32_reflected_zbc( + fold_width_crc32_reflected_zbc(acc, fold_width_high, fold_width_low), + consts.poly, + consts.mu, + ) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_block_128_zbc(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: (u64, u64)) { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - x[0] = chunk[0] ^ fold_16_zbc(x[0], coeff); - x[1] = chunk[1] ^ fold_16_zbc(x[1], coeff); - x[2] = chunk[2] ^ fold_16_zbc(x[2], coeff); - x[3] = chunk[3] ^ fold_16_zbc(x[3], coeff); - x[4] = chunk[4] ^ fold_16_zbc(x[4], coeff); - x[5] = chunk[5] ^ fold_16_zbc(x[5], coeff); - x[6] = chunk[6] ^ fold_16_zbc(x[6], coeff); - x[7] = chunk[7] ^ fold_16_zbc(x[7], coeff); - } +fn fold_block_128_zbc(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: (u64, u64)) { + x[0] = chunk[0] ^ fold_16_zbc(x[0], coeff); + x[1] = chunk[1] ^ fold_16_zbc(x[1], coeff); + x[2] = chunk[2] ^ fold_16_zbc(x[2], coeff); + x[3] = chunk[3] ^ fold_16_zbc(x[3], coeff); + x[4] = chunk[4] ^ fold_16_zbc(x[4], coeff); + x[5] = chunk[5] ^ fold_16_zbc(x[5], coeff); + x[6] = chunk[6] ^ fold_16_zbc(x[6], coeff); + x[7] = chunk[7] ^ fold_16_zbc(x[7], coeff); } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc(state: u32, first: &Block, rest: &[Block], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut x = load_block(first); - x[0].lo ^= state as u64; - - let coeff = consts.fold_128b; - for block in rest { - let chunk = load_block(block); - fold_block_128_zbc(&mut x, &chunk, coeff); - } - - fold_tail_zbc(x, consts) +fn update_simd_zbc(state: u32, first: &Block, rest: &[Block], consts: &Crc32ClmulConstants) -> u32 { + let mut x = load_block(first); + x[0].lo ^= state as u64; + + let coeff = consts.fold_128b; + for block in rest { + let chunk = load_block(block); + fold_block_128_zbc(&mut x, &chunk, coeff); } + + fold_tail_zbc(x, consts) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc_2way( - state: u32, - blocks: &[Block], - fold_256b: (u64, u64), - consts: &Crc32ClmulConstants, -) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zbc(state, first, rest, consts); - } - - let even = blocks.len() & !1usize; +fn update_simd_zbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc32ClmulConstants) -> u32 { + debug_assert!(!blocks.is_empty()); + + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zbc(state, first, rest, consts); + } - let coeff_256 = fold_256b; - let coeff_128 = consts.fold_128b; + let even = blocks.len() & !1usize; - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - s0[0].lo ^= state as u64; + let coeff_256 = fold_256b; + let coeff_128 = consts.fold_128b; - let mut i = 2usize; - while i < even { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - fold_block_128_zbc(&mut s0, &b0, coeff_256); - fold_block_128_zbc(&mut s1, &b1, coeff_256); - i = i.strict_add(2); - } + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + s0[0].lo ^= state as u64; - let mut combined = s1; - combined[0] ^= fold_16_zbc(s0[0], coeff_128); - combined[1] ^= fold_16_zbc(s0[1], coeff_128); - combined[2] ^= fold_16_zbc(s0[2], coeff_128); - combined[3] ^= fold_16_zbc(s0[3], coeff_128); - combined[4] ^= fold_16_zbc(s0[4], coeff_128); - combined[5] ^= fold_16_zbc(s0[5], coeff_128); - combined[6] ^= fold_16_zbc(s0[6], coeff_128); - combined[7] ^= fold_16_zbc(s0[7], coeff_128); - - if even != blocks.len() { - let tail = load_block(&blocks[even]); - fold_block_128_zbc(&mut combined, &tail, coeff_128); - } + let mut i = 2usize; + while i < even { + let b0 = load_block(&blocks[i]); + let b1 = load_block(&blocks[i.strict_add(1)]); + fold_block_128_zbc(&mut s0, &b0, coeff_256); + fold_block_128_zbc(&mut s1, &b1, coeff_256); + i = i.strict_add(2); + } - fold_tail_zbc(combined, consts) + let mut combined = s1; + combined[0] ^= fold_16_zbc(s0[0], coeff_128); + combined[1] ^= fold_16_zbc(s0[1], coeff_128); + combined[2] ^= fold_16_zbc(s0[2], coeff_128); + combined[3] ^= fold_16_zbc(s0[3], coeff_128); + combined[4] ^= fold_16_zbc(s0[4], coeff_128); + combined[5] ^= fold_16_zbc(s0[5], coeff_128); + combined[6] ^= fold_16_zbc(s0[6], coeff_128); + combined[7] ^= fold_16_zbc(s0[7], coeff_128); + + if even != blocks.len() { + let tail = load_block(&blocks[even]); + fold_block_128_zbc(&mut combined, &tail, coeff_128); } + + fold_tail_zbc(combined, consts) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc_4way( +fn update_simd_zbc_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], consts: &Crc32ClmulConstants, ) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); + debug_assert!(!blocks.is_empty()); - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zbc(state, first, rest, consts); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zbc(state, first, rest, consts); + } - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512 = fold_512b; - let coeff_128 = consts.fold_128b; - - let c384 = combine[0]; - let c256 = combine[1]; - let c128 = combine[2]; - - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - let mut s2 = load_block(&blocks[2]); - let mut s3 = load_block(&blocks[3]); - s0[0].lo ^= state as u64; - - let mut i = 4usize; - while i < aligned { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - let b2 = load_block(&blocks[i.strict_add(2)]); - let b3 = load_block(&blocks[i.strict_add(3)]); - fold_block_128_zbc(&mut s0, &b0, coeff_512); - fold_block_128_zbc(&mut s1, &b1, coeff_512); - fold_block_128_zbc(&mut s2, &b2, coeff_512); - fold_block_128_zbc(&mut s3, &b3, coeff_512); - i = i.strict_add(4); - } + let aligned = blocks.len().strict_div(4).strict_mul(4); + + let coeff_512 = fold_512b; + let coeff_128 = consts.fold_128b; + + let c384 = combine[0]; + let c256 = combine[1]; + let c128 = combine[2]; + + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + let mut s2 = load_block(&blocks[2]); + let mut s3 = load_block(&blocks[3]); + s0[0].lo ^= state as u64; + + let mut i = 4usize; + while i < aligned { + let b0 = load_block(&blocks[i]); + let b1 = load_block(&blocks[i.strict_add(1)]); + let b2 = load_block(&blocks[i.strict_add(2)]); + let b3 = load_block(&blocks[i.strict_add(3)]); + fold_block_128_zbc(&mut s0, &b0, coeff_512); + fold_block_128_zbc(&mut s1, &b1, coeff_512); + fold_block_128_zbc(&mut s2, &b2, coeff_512); + fold_block_128_zbc(&mut s3, &b3, coeff_512); + i = i.strict_add(4); + } - let mut combined = s3; - combined[0] ^= fold_16_zbc(s2[0], c128); - combined[1] ^= fold_16_zbc(s2[1], c128); - combined[2] ^= fold_16_zbc(s2[2], c128); - combined[3] ^= fold_16_zbc(s2[3], c128); - combined[4] ^= fold_16_zbc(s2[4], c128); - combined[5] ^= fold_16_zbc(s2[5], c128); - combined[6] ^= fold_16_zbc(s2[6], c128); - combined[7] ^= fold_16_zbc(s2[7], c128); - - combined[0] ^= fold_16_zbc(s1[0], c256); - combined[1] ^= fold_16_zbc(s1[1], c256); - combined[2] ^= fold_16_zbc(s1[2], c256); - combined[3] ^= fold_16_zbc(s1[3], c256); - combined[4] ^= fold_16_zbc(s1[4], c256); - combined[5] ^= fold_16_zbc(s1[5], c256); - combined[6] ^= fold_16_zbc(s1[6], c256); - combined[7] ^= fold_16_zbc(s1[7], c256); - - combined[0] ^= fold_16_zbc(s0[0], c384); - combined[1] ^= fold_16_zbc(s0[1], c384); - combined[2] ^= fold_16_zbc(s0[2], c384); - combined[3] ^= fold_16_zbc(s0[3], c384); - combined[4] ^= fold_16_zbc(s0[4], c384); - combined[5] ^= fold_16_zbc(s0[5], c384); - combined[6] ^= fold_16_zbc(s0[6], c384); - combined[7] ^= fold_16_zbc(s0[7], c384); - - if aligned != blocks.len() { - let tail_blocks = &blocks[aligned..]; - let Some((first, rest)) = tail_blocks.split_first() else { - return fold_tail_zbc(combined, consts); - }; - let mut x = combined; - let first = load_block(first); - fold_block_128_zbc(&mut x, &first, coeff_128); - for b in rest { - let chunk = load_block(b); - fold_block_128_zbc(&mut x, &chunk, coeff_128); - } - return fold_tail_zbc(x, consts); + let mut combined = s3; + combined[0] ^= fold_16_zbc(s2[0], c128); + combined[1] ^= fold_16_zbc(s2[1], c128); + combined[2] ^= fold_16_zbc(s2[2], c128); + combined[3] ^= fold_16_zbc(s2[3], c128); + combined[4] ^= fold_16_zbc(s2[4], c128); + combined[5] ^= fold_16_zbc(s2[5], c128); + combined[6] ^= fold_16_zbc(s2[6], c128); + combined[7] ^= fold_16_zbc(s2[7], c128); + + combined[0] ^= fold_16_zbc(s1[0], c256); + combined[1] ^= fold_16_zbc(s1[1], c256); + combined[2] ^= fold_16_zbc(s1[2], c256); + combined[3] ^= fold_16_zbc(s1[3], c256); + combined[4] ^= fold_16_zbc(s1[4], c256); + combined[5] ^= fold_16_zbc(s1[5], c256); + combined[6] ^= fold_16_zbc(s1[6], c256); + combined[7] ^= fold_16_zbc(s1[7], c256); + + combined[0] ^= fold_16_zbc(s0[0], c384); + combined[1] ^= fold_16_zbc(s0[1], c384); + combined[2] ^= fold_16_zbc(s0[2], c384); + combined[3] ^= fold_16_zbc(s0[3], c384); + combined[4] ^= fold_16_zbc(s0[4], c384); + combined[5] ^= fold_16_zbc(s0[5], c384); + combined[6] ^= fold_16_zbc(s0[6], c384); + combined[7] ^= fold_16_zbc(s0[7], c384); + + if aligned != blocks.len() { + let tail_blocks = &blocks[aligned..]; + let Some((first, rest)) = tail_blocks.split_first() else { + return fold_tail_zbc(combined, consts); + }; + let mut x = combined; + let first = load_block(first); + fold_block_128_zbc(&mut x, &first, coeff_128); + for b in rest { + let chunk = load_block(b); + fold_block_128_zbc(&mut x, &chunk, coeff_128); } - - fold_tail_zbc(combined, consts) + return fold_tail_zbc(x, consts); } + + fold_tail_zbc(combined, consts) } // ZVBC (vector carryless multiply) backend @@ -395,12 +380,9 @@ unsafe fn update_simd_zbc_4way( fn load_block_split(block: &Block) -> ([u64; 8], [u64; 8]) { let mut hi = [0u64; 8]; let mut lo = [0u64; 8]; - - let mut i = 0; - while i < 8 { - lo[i] = u64::from_le(block[i * 2]); - hi[i] = u64::from_le(block[i * 2 + 1]); - i = i.strict_add(1); + for ((hi_lane, lo_lane), &[low, high]) in hi.iter_mut().zip(&mut lo).zip(block.as_chunks::<2>().0) { + *hi_lane = u64::from_le(high); + *lo_lane = u64::from_le(low); } (hi, lo) @@ -413,7 +395,7 @@ fn load_block_split(block: &Block) -> ([u64; 8], [u64; 8]) { /// Requires RISC-V `v` + `zvbc`. #[inline] #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { +fn mul64_zvbc(a: u64, b: u64) -> Simd { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -440,74 +422,73 @@ unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - mul64_zvbc(x.low_64(), coeff_high) ^ mul64_zvbc(x.high_64(), coeff_low) - } +fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { + let (coeff_high, coeff_low) = coeff; + mul64_zvbc(x.low_64(), coeff_high) ^ mul64_zvbc(x.high_64(), coeff_low) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_width_crc32_reflected_zvbc(x: Simd, high: u64, low: u64) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let clmul = mul64_zvbc(x.low_64(), low); - let shifted = Simd::new(0, x.high_64()); - let mut state = clmul ^ shifted; +fn fold_width_crc32_reflected_zvbc(x: Simd, high: u64, low: u64) -> Simd { + let clmul = mul64_zvbc(x.low_64(), low); + let shifted = Simd::new(0, x.high_64()); + let mut state = clmul ^ shifted; - let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = mul64_zvbc(shifted_high, high); - state = clmul ^ masked; + let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = mul64_zvbc(shifted_high, high); + state = clmul ^ masked; - state - } + state } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn barrett_crc32_reflected_zvbc(x: Simd, poly: u64, mu: u64) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = mul64_zvbc(x.low_64(), mu); - let l = mul64_zvbc(t1.low_64(), poly); - (x ^ l).high_64() as u32 - } +fn barrett_crc32_reflected_zvbc(x: Simd, poly: u64, mu: u64) -> u32 { + let t1 = mul64_zvbc(x.low_64(), mu); + let l = mul64_zvbc(t1.low_64(), poly); + low_u32((x ^ l).high_64()) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut acc = Simd::new(hi[7], lo[7]); - acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), consts.tail_fold_16b[0]); - acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), consts.tail_fold_16b[1]); - acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), consts.tail_fold_16b[2]); - acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), consts.tail_fold_16b[3]); - acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), consts.tail_fold_16b[4]); - acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), consts.tail_fold_16b[5]); - acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), consts.tail_fold_16b[6]); - - let (fold_width_high, fold_width_low) = consts.fold_width; - barrett_crc32_reflected_zvbc( - fold_width_crc32_reflected_zvbc(acc, fold_width_high, fold_width_low), - consts.poly, - consts.mu, - ) - } +fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], consts: &Crc32ClmulConstants) -> u32 { + let mut acc = Simd::new(hi[7], lo[7]); + acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), consts.tail_fold_16b[0]); + acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), consts.tail_fold_16b[1]); + acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), consts.tail_fold_16b[2]); + acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), consts.tail_fold_16b[3]); + acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), consts.tail_fold_16b[4]); + acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), consts.tail_fold_16b[5]); + acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), consts.tail_fold_16b[6]); + + let (fold_width_high, fold_width_low) = consts.fold_width; + barrett_crc32_reflected_zvbc( + fold_width_crc32_reflected_zvbc(acc, fold_width_high, fold_width_low), + consts.poly, + consts.mu, + ) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_block_128_zvbc( +fn fold_block_128_zvbc( x_hi: &mut [u64; 8], x_lo: &mut [u64; 8], chunk_hi: &[u64; 8], @@ -520,7 +501,7 @@ unsafe fn fold_block_128_zvbc( unsafe { let mut offset = 0usize; while offset < 8 { - let remaining = 8 - offset; + let remaining = 8usize.strict_sub(offset); let vl: usize; asm!( "vsetvli {vl}, {avl}, e64, m1, ta, ma", @@ -560,188 +541,183 @@ unsafe fn fold_block_128_zvbc( } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc(state: u32, first: &Block, rest: &[Block], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (mut x_hi, mut x_lo) = load_block_split(first); - x_lo[0] ^= state as u64; - - let coeff_low = consts.fold_128b.1; - let coeff_high = consts.fold_128b.0; +fn update_simd_zvbc(state: u32, first: &Block, rest: &[Block], consts: &Crc32ClmulConstants) -> u32 { + let (mut x_hi, mut x_lo) = load_block_split(first); + x_lo[0] ^= state as u64; - for block in rest { - let (chunk_hi, chunk_lo) = load_block_split(block); - fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); - } + let coeff_low = consts.fold_128b.1; + let coeff_high = consts.fold_128b.0; - fold_tail_zvbc(x_hi, x_lo, consts) + for block in rest { + let (chunk_hi, chunk_lo) = load_block_split(block); + fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); } + + fold_tail_zvbc(x_hi, x_lo, consts) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_2way( - state: u32, - blocks: &[Block], - fold_256b: (u64, u64), - consts: &Crc32ClmulConstants, -) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, consts); - } - - let even = blocks.len() & !1usize; +fn update_simd_zvbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc32ClmulConstants) -> u32 { + debug_assert!(!blocks.is_empty()); + + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zvbc(state, first, rest, consts); + } - let coeff_256_low = fold_256b.1; - let coeff_256_high = fold_256b.0; - let coeff_128_low = consts.fold_128b.1; - let coeff_128_high = consts.fold_128b.0; + let even = blocks.len() & !1usize; - let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); - s0_lo[0] ^= state as u64; + let coeff_256_low = fold_256b.1; + let coeff_256_high = fold_256b.0; + let coeff_128_low = consts.fold_128b.1; + let coeff_128_high = consts.fold_128b.0; - let mut i = 2usize; - while i < even { - let (b0_hi, b0_lo) = load_block_split(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); - i = i.strict_add(2); - } + let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); + let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); + s0_lo[0] ^= state as u64; - // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). Use the same trick as CRC64: - // fold each stream into the current combined value using `chunk = combined`. - let mut combined_hi = s1_hi; - let mut combined_lo = s1_lo; + let mut i = 2usize; + while i < even { + let (b0_hi, b0_lo) = load_block_split(&blocks[i]); + let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); + i = i.strict_add(2); + } + // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). Use the same trick as CRC64: + // fold each stream into the current combined value using `chunk = combined`. + let mut combined_hi = s1_hi; + let mut combined_lo = s1_lo; + + fold_block_128_zvbc( + &mut s0_hi, + &mut s0_lo, + &combined_hi, + &combined_lo, + coeff_128_low, + coeff_128_high, + ); + combined_hi = s0_hi; + combined_lo = s0_lo; + + if even != blocks.len() { + let (tail_hi, tail_lo) = load_block_split(&blocks[even]); fold_block_128_zvbc( - &mut s0_hi, - &mut s0_lo, - &combined_hi, - &combined_lo, + &mut combined_hi, + &mut combined_lo, + &tail_hi, + &tail_lo, coeff_128_low, coeff_128_high, ); - combined_hi = s0_hi; - combined_lo = s0_lo; - - if even != blocks.len() { - let (tail_hi, tail_lo) = load_block_split(&blocks[even]); - fold_block_128_zvbc( - &mut combined_hi, - &mut combined_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } - - fold_tail_zvbc(combined_hi, combined_lo, consts) } + + fold_tail_zvbc(combined_hi, combined_lo, consts) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_4way( +fn update_simd_zvbc_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], consts: &Crc32ClmulConstants, ) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); + debug_assert!(!blocks.is_empty()); - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, consts); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zvbc(state, first, rest, consts); + } - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512_low = fold_512b.1; - let coeff_512_high = fold_512b.0; - let coeff_128_low = consts.fold_128b.1; - let coeff_128_high = consts.fold_128b.0; - - let c384_low = combine[0].1; - let c384_high = combine[0].0; - let c256_low = combine[1].1; - let c256_high = combine[1].0; - let c128_low = combine[2].1; - let c128_high = combine[2].0; - - let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); - let (mut s2_hi, mut s2_lo) = load_block_split(&blocks[2]); - let (mut s3_hi, mut s3_lo) = load_block_split(&blocks[3]); - s0_lo[0] ^= state as u64; - - let mut i = 4usize; - while i < aligned { - let (b0_hi, b0_lo) = load_block_split(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); - let (b2_hi, b2_lo) = load_block_split(&blocks[i.strict_add(2)]); - let (b3_hi, b3_lo) = load_block_split(&blocks[i.strict_add(3)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); - i = i.strict_add(4); - } + let aligned = blocks.len().strict_div(4).strict_mul(4); + + let coeff_512_low = fold_512b.1; + let coeff_512_high = fold_512b.0; + let coeff_128_low = consts.fold_128b.1; + let coeff_128_high = consts.fold_128b.0; + + let c384_low = combine[0].1; + let c384_high = combine[0].0; + let c256_low = combine[1].1; + let c256_high = combine[1].0; + let c128_low = combine[2].1; + let c128_high = combine[2].0; + + let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); + let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); + let (mut s2_hi, mut s2_lo) = load_block_split(&blocks[2]); + let (mut s3_hi, mut s3_lo) = load_block_split(&blocks[3]); + s0_lo[0] ^= state as u64; + + let mut i = 4usize; + while i < aligned { + let (b0_hi, b0_lo) = load_block_split(&blocks[i]); + let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); + let (b2_hi, b2_lo) = load_block_split(&blocks[i.strict_add(2)]); + let (b3_hi, b3_lo) = load_block_split(&blocks[i.strict_add(3)]); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); + i = i.strict_add(4); + } - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - let mut combined_hi = s3_hi; - let mut combined_lo = s3_lo; - - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &combined_hi, &combined_lo, c128_low, c128_high); - combined_hi = s2_hi; - combined_lo = s2_lo; - - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &combined_hi, &combined_lo, c256_low, c256_high); - combined_hi = s1_hi; - combined_lo = s1_lo; - - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &combined_hi, &combined_lo, c384_low, c384_high); - combined_hi = s0_hi; - combined_lo = s0_lo; - - for block in &blocks[aligned..] { - let (tail_hi, tail_lo) = load_block_split(block); - fold_block_128_zvbc( - &mut combined_hi, - &mut combined_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + let mut combined_hi = s3_hi; + let mut combined_lo = s3_lo; + + fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &combined_hi, &combined_lo, c128_low, c128_high); + combined_hi = s2_hi; + combined_lo = s2_lo; + + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &combined_hi, &combined_lo, c256_low, c256_high); + combined_hi = s1_hi; + combined_lo = s1_lo; + + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &combined_hi, &combined_lo, c384_low, c384_high); + combined_hi = s0_hi; + combined_lo = s0_lo; - fold_tail_zvbc(combined_hi, combined_lo, consts) + for block in &blocks[aligned..] { + let (tail_hi, tail_lo) = load_block_split(block); + fold_block_128_zvbc( + &mut combined_hi, + &mut combined_lo, + &tail_hi, + &tail_lo, + coeff_128_low, + coeff_128_high, + ); } + + fold_tail_zvbc(combined_hi, combined_lo, consts) } // Public kernels (IEEE + CRC32C) +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc32_zbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc32_zbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -769,15 +745,18 @@ unsafe fn crc32_zbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc32_zbc_nway( +fn crc32_zbc_nway( mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants, stream: &Crc32StreamConstants, ) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { debug_assert!(N == 2 || N == 4); let (left, middle, right) = bytes.align_to::(); @@ -804,10 +783,13 @@ unsafe fn crc32_zbc_nway( } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc32c_zbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc32c_zbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -832,15 +814,18 @@ unsafe fn crc32c_zbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc32c_zbc_nway( +fn crc32c_zbc_nway( mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants, stream: &Crc32StreamConstants, ) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { debug_assert!(N == 2 || N == 4); let (left, middle, right) = bytes.align_to::(); @@ -867,10 +852,13 @@ unsafe fn crc32c_zbc_nway( } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc32_zvbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc32_zvbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -895,15 +883,18 @@ unsafe fn crc32_zvbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc32_zvbc_nway( +fn crc32_zvbc_nway( mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants, stream: &Crc32StreamConstants, ) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { debug_assert!(N == 2 || N == 4); let (left, middle, right) = bytes.align_to::(); @@ -930,10 +921,13 @@ unsafe fn crc32_zvbc_nway( } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc32c_zvbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc32c_zvbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -958,15 +952,18 @@ unsafe fn crc32c_zvbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc32c_zvbc_nway( +fn crc32c_zvbc_nway( mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants, stream: &Crc32StreamConstants, ) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { debug_assert!(N == 2 || N == 4); let (left, middle, right) = bytes.align_to::(); @@ -996,15 +993,14 @@ unsafe fn crc32c_zvbc_nway( // Safe wrappers (dispatcher entrypoints) #[inline] -pub fn crc32_ieee_zbc_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_zbc_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies ZBC before selecting this kernel. unsafe { crc32_zbc(crc, data, &super::clmul::CRC32_IEEE_CLMUL) } } #[inline] -pub fn crc32_ieee_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32_ieee_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc32_zbc_nway::<2>( crc, @@ -1016,9 +1012,8 @@ pub fn crc32_ieee_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32_ieee_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc32_zbc_nway::<4>( crc, @@ -1030,15 +1025,14 @@ pub fn crc32_ieee_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_zvbc_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_zvbc_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies V+ZVBC before selecting this kernel. unsafe { crc32_zvbc(crc, data, &super::clmul::CRC32_IEEE_CLMUL) } } #[inline] -pub fn crc32_ieee_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32_ieee_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies V and Zvbc before selecting this kernel. unsafe { crc32_zvbc_nway::<2>( crc, @@ -1050,9 +1044,8 @@ pub fn crc32_ieee_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32_ieee_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies V and Zvbc before selecting this kernel. unsafe { crc32_zvbc_nway::<4>( crc, @@ -1064,44 +1057,38 @@ pub fn crc32_ieee_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32c_zbc_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32c_zbc_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc32c_zbc(crc, data, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32c_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc32c_zbc_nway::<2>(crc, data, &super::clmul::CRC32C_CLMUL, &super::clmul::CRC32C_STREAM) } } #[inline] -pub fn crc32c_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32c_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc32c_zbc_nway::<4>(crc, data, &super::clmul::CRC32C_CLMUL, &super::clmul::CRC32C_STREAM) } } #[inline] -pub fn crc32c_zvbc_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32c_zvbc_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies V and Zvbc before selecting this kernel. unsafe { crc32c_zvbc(crc, data, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32c_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies V and Zvbc before selecting this kernel. unsafe { crc32c_zvbc_nway::<2>(crc, data, &super::clmul::CRC32C_CLMUL, &super::clmul::CRC32C_STREAM) } } #[inline] -pub fn crc32c_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32c_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies V and Zvbc before selecting this kernel. unsafe { crc32c_zvbc_nway::<4>(crc, data, &super::clmul::CRC32C_CLMUL, &super::clmul::CRC32C_STREAM) } } diff --git a/src/checksum/crc32/s390x.rs b/src/checksum/crc32/s390x.rs index 9c3fed1d..0a3754b9 100644 --- a/src/checksum/crc32/s390x.rs +++ b/src/checksum/crc32/s390x.rs @@ -8,19 +8,15 @@ //! Uses `unsafe` for s390x vector + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] -// This module is intrinsics-heavy; unsafe blocks are per-function with SAFETY justifications. use core::{ arch::asm, - mem::MaybeUninit, ops::{BitAnd, BitXor, BitXorAssign}, simd::i64x2, }; +use crate::checksum::common::low_u32; + use super::clmul::{Crc32ClmulConstants, Crc32StreamConstants}; type Block = [u64; 16]; // 128 bytes (8×16B lanes) @@ -78,8 +74,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn vgfm(a: i64x2, b: i64x2) -> i64x2 { + fn vgfm(a: i64x2, b: i64x2) -> i64x2 { // SAFETY: Caller guarantees the s390x vector facility is available // (verified by dispatch). The VGFM instruction operates on pure register // values with no memory access. @@ -97,63 +96,57 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Caller guarantees the s390x vector facility is available. - // vgfm operates on pure register values. - unsafe { - let va = Self::new(0, a); - let vb = Self::new(0, b); - Self(Self::vgfm(va.0, vb.0)) - } + fn mul64(a: u64, b: u64) -> Self { + let va = Self::new(0, a); + let vb = Self::new(0, b); + Self(Self::vgfm(va.0, vb.0)) } /// Fold 16 bytes (reflected CRC32 folding primitive): /// `self.low ⊗ coeff.high ⊕ self.high ⊗ coeff.low`. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_16(self, coeff: Self) -> Self { - // SAFETY: Caller guarantees the s390x vector facility is available. - // vgfm operates on pure register values after lane swap. - unsafe { - // Like VPMSUMD: VGFM performs a per-lane carryless multiply and XORs the lane products. - // The CRC32 folding primitive needs cross terms, so swap coefficient lanes. - Self(Self::vgfm(self.0, coeff.swap_lanes().0)) - } + fn fold_16(self, coeff: Self) -> Self { + Self(Self::vgfm(self.0, coeff.swap_lanes().0)) } /// Fold 16B → CRC32 width (reflected), returning an intermediate 128-bit state. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_width_crc32_reflected(self, high: u64, low: u64) -> Self { - // SAFETY: Caller guarantees the s390x vector facility is available. - // All operations are pure register computations via mul64/vgfm. - unsafe { - let clmul = Self::mul64(self.low_64(), low); - let shifted = Self::new(0, self.high_64()); - let mut state = clmul ^ shifted; - - let mask2 = Self::new(0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_0000_0000); - let masked = state & mask2; - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = Self::mul64(shifted_high, high); - state = clmul ^ masked; - - state - } + fn fold_width_crc32_reflected(self, high: u64, low: u64) -> Self { + let clmul = Self::mul64(self.low_64(), low); + let shifted = Self::new(0, self.high_64()); + let mut state = clmul ^ shifted; + + let mask2 = Self::new(0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_0000_0000); + let masked = state & mask2; + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = Self::mul64(shifted_high, high); + state = clmul ^ masked; + + state } /// Barrett reduction for reflected CRC32; returns the updated (pre-inverted) CRC. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn barrett_crc32_reflected(self, poly: u64, mu: u64) -> u32 { - // SAFETY: Caller guarantees the s390x vector facility is available. - // mul64 and XOR are pure register computations. - unsafe { - let t1 = Self::mul64(self.low_64(), mu); - let l = Self::mul64(t1.low_64(), poly); - (self ^ l).high_64() as u32 - } + fn barrett_crc32_reflected(self, poly: u64, mu: u64) -> u32 { + let t1 = Self::mul64(self.low_64(), mu); + let l = Self::mul64(t1.low_64(), poly); + low_u32((self ^ l).high_64()) } } @@ -161,149 +154,133 @@ impl Simd { #[inline(always)] fn load_block(block: &Block) -> [Simd; 8] { - let mut out = MaybeUninit::<[Simd; 8]>::uninit(); - let base = out.as_mut_ptr().cast::(); - - let mut i = 0; - while i < 8 { - let low = u64::from_le(block[i * 2]); - let high = u64::from_le(block[i * 2 + 1]); - // SAFETY: `base` points to a `[Simd; 8]` buffer and `i` is in-bounds. - unsafe { - base.add(i).write(Simd::new(high, low)); - } - i = i.strict_add(1); + let mut out = [Simd::new(0, 0); 8]; + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new(u64::from_le(high), u64::from_le(low)); } - - // SAFETY: all 8 elements are initialized above. - unsafe { out.assume_init() } + out } // Folding helpers -#[inline(always)] -unsafe fn fold_tail(x: [Simd; 8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold_16 / fold_width / barrett calls are pure VGFM register computations. - // Array indexing is over fixed-size [Simd; 8] with constant indices 0..7. - unsafe { - let c0 = Simd::new(consts.tail_fold_16b[0].0, consts.tail_fold_16b[0].1); - let c1 = Simd::new(consts.tail_fold_16b[1].0, consts.tail_fold_16b[1].1); - let c2 = Simd::new(consts.tail_fold_16b[2].0, consts.tail_fold_16b[2].1); - let c3 = Simd::new(consts.tail_fold_16b[3].0, consts.tail_fold_16b[3].1); - let c4 = Simd::new(consts.tail_fold_16b[4].0, consts.tail_fold_16b[4].1); - let c5 = Simd::new(consts.tail_fold_16b[5].0, consts.tail_fold_16b[5].1); - let c6 = Simd::new(consts.tail_fold_16b[6].0, consts.tail_fold_16b[6].1); - - let mut acc = x[7]; - acc ^= x[0].fold_16(c0); - acc ^= x[1].fold_16(c1); - acc ^= x[2].fold_16(c2); - acc ^= x[3].fold_16(c3); - acc ^= x[4].fold_16(c4); - acc ^= x[5].fold_16(c5); - acc ^= x[6].fold_16(c6); - - let (fold_width_high, fold_width_low) = consts.fold_width; - let state = acc.fold_width_crc32_reflected(fold_width_high, fold_width_low); - state.barrett_crc32_reflected(consts.poly, consts.mu) - } +#[inline] +/// # Safety +/// +/// Requires the s390x vector facility. +#[target_feature(enable = "vector")] +fn fold_tail(x: [Simd; 8], consts: &Crc32ClmulConstants) -> u32 { + let c0 = Simd::new(consts.tail_fold_16b[0].0, consts.tail_fold_16b[0].1); + let c1 = Simd::new(consts.tail_fold_16b[1].0, consts.tail_fold_16b[1].1); + let c2 = Simd::new(consts.tail_fold_16b[2].0, consts.tail_fold_16b[2].1); + let c3 = Simd::new(consts.tail_fold_16b[3].0, consts.tail_fold_16b[3].1); + let c4 = Simd::new(consts.tail_fold_16b[4].0, consts.tail_fold_16b[4].1); + let c5 = Simd::new(consts.tail_fold_16b[5].0, consts.tail_fold_16b[5].1); + let c6 = Simd::new(consts.tail_fold_16b[6].0, consts.tail_fold_16b[6].1); + + let mut acc = x[7]; + acc ^= x[0].fold_16(c0); + acc ^= x[1].fold_16(c1); + acc ^= x[2].fold_16(c2); + acc ^= x[3].fold_16(c3); + acc ^= x[4].fold_16(c4); + acc ^= x[5].fold_16(c5); + acc ^= x[6].fold_16(c6); + + let (fold_width_high, fold_width_low) = consts.fold_width; + let state = acc.fold_width_crc32_reflected(fold_width_high, fold_width_low); + state.barrett_crc32_reflected(consts.poly, consts.mu) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold_16 calls are pure VGFM register computations. - // Array indexing is over fixed-size [Simd; 8] with constant indices 0..7. - unsafe { - x[0] = chunk[0] ^ x[0].fold_16(coeff); - x[1] = chunk[1] ^ x[1].fold_16(coeff); - x[2] = chunk[2] ^ x[2].fold_16(coeff); - x[3] = chunk[3] ^ x[3].fold_16(coeff); - x[4] = chunk[4] ^ x[4].fold_16(coeff); - x[5] = chunk[5] ^ x[5].fold_16(coeff); - x[6] = chunk[6] ^ x[6].fold_16(coeff); - x[7] = chunk[7] ^ x[7].fold_16(coeff); - } +fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { + x[0] = chunk[0] ^ x[0].fold_16(coeff); + x[1] = chunk[1] ^ x[1].fold_16(coeff); + x[2] = chunk[2] ^ x[2].fold_16(coeff); + x[3] = chunk[3] ^ x[3].fold_16(coeff); + x[4] = chunk[4] ^ x[4].fold_16(coeff); + x[5] = chunk[5] ^ x[5].fold_16(coeff); + x[6] = chunk[6] ^ x[6].fold_16(coeff); + x[7] = chunk[7] ^ x[7].fold_16(coeff); } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd(state: u32, first: &Block, rest: &[Block], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold operations are pure VGFM register computations. - // Block iteration is bounded by the slice length; no out-of-bounds access. - unsafe { - let mut x = load_block(first); +fn update_simd(state: u32, first: &Block, rest: &[Block], consts: &Crc32ClmulConstants) -> u32 { + let mut x = load_block(first); - // XOR initial CRC into the first 16-byte lane (low 32 bits). - x[0] ^= Simd::new(0, state as u64); + // XOR initial CRC into the first 16-byte lane (low 32 bits). + x[0] ^= Simd::new(0, state as u64); - let coeff = Simd::new(consts.fold_128b.0, consts.fold_128b.1); - for block in rest { - let chunk = load_block(block); - fold_block_128(&mut x, &chunk, coeff); - } - - fold_tail(x, consts) + let coeff = Simd::new(consts.fold_128b.0, consts.fold_128b.1); + for block in rest { + let chunk = load_block(block); + fold_block_128(&mut x, &chunk, coeff); } + + fold_tail(x, consts) } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc32ClmulConstants) -> u32 { +fn update_simd_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc32ClmulConstants) -> u32 { debug_assert!(!blocks.is_empty()); - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold operations are pure VGFM register computations. - // Loop index `i` advances by 2 and is bounded by `even` (≤ blocks.len()), - // so `blocks[i]` and `blocks[i + 1]` are always in-bounds. - unsafe { - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } - - let even = blocks.len() & !1usize; + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd(state, first, rest, consts); + } - let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); - let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); + let even = blocks.len() & !1usize; - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - s0[0] ^= Simd::new(0, state as u64); + let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); + let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); - let mut i = 2usize; - while i < even { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - fold_block_128(&mut s0, &b0, coeff_256); - fold_block_128(&mut s1, &b1, coeff_256); - i = i.strict_add(2); - } + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + s0[0] ^= Simd::new(0, state as u64); - let mut combined = s1; - combined[0] ^= s0[0].fold_16(coeff_128); - combined[1] ^= s0[1].fold_16(coeff_128); - combined[2] ^= s0[2].fold_16(coeff_128); - combined[3] ^= s0[3].fold_16(coeff_128); - combined[4] ^= s0[4].fold_16(coeff_128); - combined[5] ^= s0[5].fold_16(coeff_128); - combined[6] ^= s0[6].fold_16(coeff_128); - combined[7] ^= s0[7].fold_16(coeff_128); - - if even != blocks.len() { - let tail = load_block(&blocks[even]); - fold_block_128(&mut combined, &tail, coeff_128); - } + let mut i = 2usize; + while i < even { + let b0 = load_block(&blocks[i]); + let b1 = load_block(&blocks[i.strict_add(1)]); + fold_block_128(&mut s0, &b0, coeff_256); + fold_block_128(&mut s1, &b1, coeff_256); + i = i.strict_add(2); + } - fold_tail(combined, consts) + let mut combined = s1; + combined[0] ^= s0[0].fold_16(coeff_128); + combined[1] ^= s0[1].fold_16(coeff_128); + combined[2] ^= s0[2].fold_16(coeff_128); + combined[3] ^= s0[3].fold_16(coeff_128); + combined[4] ^= s0[4].fold_16(coeff_128); + combined[5] ^= s0[5].fold_16(coeff_128); + combined[6] ^= s0[6].fold_16(coeff_128); + combined[7] ^= s0[7].fold_16(coeff_128); + + if even != blocks.len() { + let tail = load_block(&blocks[even]); + fold_block_128(&mut combined, &tail, coeff_128); } + + fold_tail(combined, consts) } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_4way( +fn update_simd_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), @@ -312,99 +289,96 @@ unsafe fn update_simd_4way( ) -> u32 { debug_assert!(!blocks.is_empty()); - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold operations are pure VGFM register computations. - // Loop index `i` advances by 4 and is bounded by `aligned` (≤ blocks.len()), - // so `blocks[i..i+3]` are always in-bounds. Tail processing uses split_first. - unsafe { - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd(state, first, rest, consts); + } - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); - let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); - let c384 = Simd::new(combine[0].0, combine[0].1); - let c256 = Simd::new(combine[1].0, combine[1].1); - let c128 = Simd::new(combine[2].0, combine[2].1); - - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - let mut s2 = load_block(&blocks[2]); - let mut s3 = load_block(&blocks[3]); - s0[0] ^= Simd::new(0, state as u64); - - let mut i = 4usize; - while i < aligned { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - let b2 = load_block(&blocks[i.strict_add(2)]); - let b3 = load_block(&blocks[i.strict_add(3)]); - fold_block_128(&mut s0, &b0, coeff_512); - fold_block_128(&mut s1, &b1, coeff_512); - fold_block_128(&mut s2, &b2, coeff_512); - fold_block_128(&mut s3, &b3, coeff_512); - i = i.strict_add(4); - } + let aligned = blocks.len().strict_div(4).strict_mul(4); + + let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); + let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); + let c384 = Simd::new(combine[0].0, combine[0].1); + let c256 = Simd::new(combine[1].0, combine[1].1); + let c128 = Simd::new(combine[2].0, combine[2].1); + + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + let mut s2 = load_block(&blocks[2]); + let mut s3 = load_block(&blocks[3]); + s0[0] ^= Simd::new(0, state as u64); + + let mut i = 4usize; + while i < aligned { + let b0 = load_block(&blocks[i]); + let b1 = load_block(&blocks[i.strict_add(1)]); + let b2 = load_block(&blocks[i.strict_add(2)]); + let b3 = load_block(&blocks[i.strict_add(3)]); + fold_block_128(&mut s0, &b0, coeff_512); + fold_block_128(&mut s1, &b1, coeff_512); + fold_block_128(&mut s2, &b2, coeff_512); + fold_block_128(&mut s3, &b3, coeff_512); + i = i.strict_add(4); + } - let mut combined = s3; - combined[0] ^= s2[0].fold_16(c128); - combined[1] ^= s2[1].fold_16(c128); - combined[2] ^= s2[2].fold_16(c128); - combined[3] ^= s2[3].fold_16(c128); - combined[4] ^= s2[4].fold_16(c128); - combined[5] ^= s2[5].fold_16(c128); - combined[6] ^= s2[6].fold_16(c128); - combined[7] ^= s2[7].fold_16(c128); - - combined[0] ^= s1[0].fold_16(c256); - combined[1] ^= s1[1].fold_16(c256); - combined[2] ^= s1[2].fold_16(c256); - combined[3] ^= s1[3].fold_16(c256); - combined[4] ^= s1[4].fold_16(c256); - combined[5] ^= s1[5].fold_16(c256); - combined[6] ^= s1[6].fold_16(c256); - combined[7] ^= s1[7].fold_16(c256); - - combined[0] ^= s0[0].fold_16(c384); - combined[1] ^= s0[1].fold_16(c384); - combined[2] ^= s0[2].fold_16(c384); - combined[3] ^= s0[3].fold_16(c384); - combined[4] ^= s0[4].fold_16(c384); - combined[5] ^= s0[5].fold_16(c384); - combined[6] ^= s0[6].fold_16(c384); - combined[7] ^= s0[7].fold_16(c384); - - if aligned != blocks.len() { - let tail_blocks = &blocks[aligned..]; - let Some((first, rest)) = tail_blocks.split_first() else { - return fold_tail(combined, consts); - }; - let mut x = combined; - let first = load_block(first); - fold_block_128(&mut x, &first, coeff_128); - for b in rest { - let chunk = load_block(b); - fold_block_128(&mut x, &chunk, coeff_128); - } - return fold_tail(x, consts); + let mut combined = s3; + combined[0] ^= s2[0].fold_16(c128); + combined[1] ^= s2[1].fold_16(c128); + combined[2] ^= s2[2].fold_16(c128); + combined[3] ^= s2[3].fold_16(c128); + combined[4] ^= s2[4].fold_16(c128); + combined[5] ^= s2[5].fold_16(c128); + combined[6] ^= s2[6].fold_16(c128); + combined[7] ^= s2[7].fold_16(c128); + + combined[0] ^= s1[0].fold_16(c256); + combined[1] ^= s1[1].fold_16(c256); + combined[2] ^= s1[2].fold_16(c256); + combined[3] ^= s1[3].fold_16(c256); + combined[4] ^= s1[4].fold_16(c256); + combined[5] ^= s1[5].fold_16(c256); + combined[6] ^= s1[6].fold_16(c256); + combined[7] ^= s1[7].fold_16(c256); + + combined[0] ^= s0[0].fold_16(c384); + combined[1] ^= s0[1].fold_16(c384); + combined[2] ^= s0[2].fold_16(c384); + combined[3] ^= s0[3].fold_16(c384); + combined[4] ^= s0[4].fold_16(c384); + combined[5] ^= s0[5].fold_16(c384); + combined[6] ^= s0[6].fold_16(c384); + combined[7] ^= s0[7].fold_16(c384); + + if aligned != blocks.len() { + let tail_blocks = &blocks[aligned..]; + let Some((first, rest)) = tail_blocks.split_first() else { + return fold_tail(combined, consts); + }; + let mut x = combined; + let first = load_block(first); + fold_block_128(&mut x, &first, coeff_128); + for b in rest { + let chunk = load_block(b); + fold_block_128(&mut x, &chunk, coeff_128); } - - fold_tail(combined, consts) + return fold_tail(x, consts); } + + fold_tail(combined, consts) } // Public kernels (IEEE + CRC32C) #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc32_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices; update_simd operates on aligned Block data. +fn crc32_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -418,10 +392,13 @@ unsafe fn crc32_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) -> u } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc32c_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices; update_simd operates on aligned Block data. +fn crc32c_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -435,16 +412,19 @@ unsafe fn crc32c_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) -> } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc32_kernel_nway( +fn crc32_kernel_nway( crc: u32, data: &[u8], stream: &Crc32StreamConstants, consts: &Crc32ClmulConstants, ) -> u32 { debug_assert!(N == 2 || N == 4); - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices; update_simd_* operate on aligned Block data. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -461,16 +441,19 @@ unsafe fn crc32_kernel_nway( } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc32c_kernel_nway( +fn crc32c_kernel_nway( crc: u32, data: &[u8], stream: &Crc32StreamConstants, consts: &Crc32ClmulConstants, ) -> u32 { debug_assert!(N == 2 || N == 4); - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices; update_simd_* operate on aligned Block data. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -489,13 +472,13 @@ unsafe fn crc32c_kernel_nway( // Safe wrappers (dispatcher entrypoints) #[inline] -pub fn crc32_ieee_vgfm_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vgfm_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies vector support before selecting this kernel. unsafe { crc32_kernel(crc, data, &super::clmul::CRC32_IEEE_CLMUL) } } #[inline] -pub fn crc32_ieee_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies vector support before selecting this kernel. unsafe { crc32_kernel_nway::<2>( @@ -508,7 +491,7 @@ pub fn crc32_ieee_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies vector support before selecting this kernel. unsafe { crc32_kernel_nway::<4>( @@ -521,19 +504,19 @@ pub fn crc32_ieee_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32c_vgfm_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vgfm_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies vector support before selecting this kernel. unsafe { crc32c_kernel(crc, data, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies vector support before selecting this kernel. unsafe { crc32c_kernel_nway::<2>(crc, data, &super::clmul::CRC32C_STREAM, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies vector support before selecting this kernel. unsafe { crc32c_kernel_nway::<4>(crc, data, &super::clmul::CRC32C_STREAM, &super::clmul::CRC32C_CLMUL) } } diff --git a/src/checksum/crc32/x86_64.rs b/src/checksum/crc32/x86_64.rs index ea31b094..27654cce 100644 --- a/src/checksum/crc32/x86_64.rs +++ b/src/checksum/crc32/x86_64.rs @@ -4,11 +4,9 @@ //! //! Uses `unsafe` for x86 SIMD intrinsics. Callers must ensure SSE4.2 is //! available before executing the accelerated path (the dispatcher does this). -#![allow(unsafe_code)] // SAFETY: This module is intrinsics-heavy and uses tight, invariant-driven indexing // (e.g. fixed-size lanes and chunked block processing) where bounds are proven by // control flow; Clippy cannot always see these invariants. -#![allow(clippy::indexing_slicing)] use core::{ arch::x86_64::*, @@ -70,10 +68,13 @@ fn pow_shift8_crc32c(len_bytes: usize) -> Gf2Matrix32 { /// CRC-32C update using SSE4.2 `crc32` instruction. /// /// `crc` is the current state (pre-inverted). +/// +/// # Safety +/// +/// The current CPU must support SSE4.2. #[inline] #[target_feature(enable = "sse4.2")] unsafe fn crc32c_sse42(crc: u32, data: &[u8]) -> u32 { - // SAFETY: SSE4.2 intrinsics are available via this function's #[target_feature] attribute. let mut state64 = crc as u64; let (chunks8, tail8) = data.as_chunks::<8>(); @@ -81,7 +82,8 @@ unsafe fn crc32c_sse42(crc: u32, data: &[u8]) -> u32 { state64 = _mm_crc32_u64(state64, u64::from_le_bytes(*chunk)); } - let mut state = state64 as u32; + let [state0, state1, state2, state3, ..] = state64.to_le_bytes(); + let mut state = u32::from_le_bytes([state0, state1, state2, state3]); let (chunks4, tail4) = tail8.as_chunks::<4>(); for chunk in chunks4 { @@ -102,18 +104,25 @@ unsafe fn crc32c_sse42(crc: u32, data: &[u8]) -> u32 { /// Safe wrapper for CRC-32C SSE4.2 kernel. #[inline] -pub fn crc32c_sse42_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_sse42_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSE4.2 before selecting this kernel. unsafe { crc32c_sse42(crc, data) } } // CRC-32C HWCRC Multi-stream Kernels (SSE4.2) +/// Computes CRC-32C with the private 2-, 4-, 7-, or 8-lane SSE4.2 kernels. +/// +/// # Safety +/// +/// The current CPU must support SSE4.2. #[inline] #[target_feature(enable = "sse4.2")] unsafe fn crc32c_sse42_nway(crc: u32, data: &[u8]) -> u32 { - // SAFETY: SSE4.2 intrinsics are available via this function's #[target_feature] attribute. - // Pointer arithmetic and get_unchecked are bounded by `chunk_len` and `len` derived from `data`. + // SAFETY: The target-feature contract permits SSE4.2 intrinsics. For every lane, `base` starts + // within `N * chunk_len`; the eight-byte path additionally proves `i + 8 <= chunk_len`, and the + // byte path proves `i < chunk_len`. The tail starts at `N * chunk_len <= len`. The shared slice + // supplies initialized readable bytes, and `read_unaligned` imposes no alignment requirement. unsafe { debug_assert!(N == 2 || N == 4 || N == 7 || N == 8); @@ -123,7 +132,7 @@ unsafe fn crc32c_sse42_nway(crc: u32, data: &[u8]) -> u32 { } // Use even-sized chunking (leave tail on the last stream). - let chunk_len = len / N; + let chunk_len = len.strict_div(N); let mut lanes = [!0u32; N]; // Process a striped prefix where all lanes have full `chunk_len`. @@ -134,10 +143,12 @@ unsafe fn crc32c_sse42_nway(crc: u32, data: &[u8]) -> u32 { let mut lane_idx: usize = 0; while lane_idx < N { let base = lane_idx.strict_mul(chunk_len).strict_add(i); - lanes[lane_idx] = _mm_crc32_u64( + let next = _mm_crc32_u64( lanes[lane_idx] as u64, - ptr::read_unaligned(data.as_ptr().add(base) as *const u64), - ) as u32; + ptr::read_unaligned(data.as_ptr().add(base).cast::()), + ); + let [b0, b1, b2, b3, _, _, _, _] = next.to_le_bytes(); + lanes[lane_idx] = u32::from_le_bytes([b0, b1, b2, b3]); lane_idx = lane_idx.strict_add(1); } i = i.strict_add(8); @@ -154,8 +165,9 @@ unsafe fn crc32c_sse42_nway(crc: u32, data: &[u8]) -> u32 { // Finish the tail on the last lane (remaining bytes after N*chunk_len). let tail_start = chunk_len.strict_mul(N); + let last_lane = N.strict_sub(1); if tail_start < len { - lanes[N - 1] = crc32c_sse42(lanes[N - 1], data.get_unchecked(tail_start..)); + lanes[last_lane] = crc32c_sse42(lanes[last_lane], data.get_unchecked(tail_start..)); } // Compute CRC(data) under the standard initial state, then append it to `crc`. @@ -188,25 +200,25 @@ unsafe fn crc32c_sse42_nway(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32c_sse42_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_sse42_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSE4.2 before selecting this kernel. unsafe { crc32c_sse42_nway::<2>(crc, data) } } #[inline] -pub fn crc32c_sse42_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_sse42_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSE4.2 before selecting this kernel. unsafe { crc32c_sse42_nway::<4>(crc, data) } } #[inline] -pub fn crc32c_sse42_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_sse42_7way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSE4.2 before selecting this kernel. unsafe { crc32c_sse42_nway::<7>(crc, data) } } #[inline] -pub fn crc32c_sse42_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_sse42_8way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSE4.2 before selecting this kernel. unsafe { crc32c_sse42_nway::<8>(crc, data) } } @@ -216,31 +228,47 @@ pub fn crc32c_sse42_8way_safe(crc: u32, data: &[u8]) -> u32 { // Derived from Corsix `fast-crc32` generator output (v4s3x3 / v3x2 families). // These kernels fuse hardware CRC instructions with carryless multiply folding. +/// Multiplies the low 64-bit lanes of two vectors over GF(2). +/// +/// # Safety +/// +/// The current CPU must support PCLMULQDQ. #[inline] #[target_feature(enable = "pclmulqdq")] unsafe fn clmul_lo_sse(a: __m128i, b: __m128i) -> __m128i { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. _mm_clmulepi64_si128::<0x00>(a, b) } +/// Multiplies the high 64-bit lanes of two vectors over GF(2). +/// +/// # Safety +/// +/// The current CPU must support PCLMULQDQ. #[inline] #[target_feature(enable = "pclmulqdq")] unsafe fn clmul_hi_sse(a: __m128i, b: __m128i) -> __m128i { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. _mm_clmulepi64_si128::<0x11>(a, b) } +/// Multiplies two 32-bit scalars over GF(2) and returns the 64-bit product. +/// +/// # Safety +/// +/// The current CPU must support PCLMULQDQ. #[inline] #[target_feature(enable = "pclmulqdq")] unsafe fn clmul_scalar_sse(a: u32, b: u32) -> __m128i { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. _mm_clmulepi64_si128::<0x00>(_mm_cvtsi32_si128(a.cast_signed()), _mm_cvtsi32_si128(b.cast_signed())) } +/// Extracts the low lane for index zero and the high lane otherwise. +/// +/// # Safety +/// +/// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn mm_extract_epi64(val: __m128i, idx: i32) -> u64 { - // SAFETY: SSE2 intrinsics are available via this function's #[target_feature] attribute. if idx == 0 { _mm_cvtsi128_si64(val).cast_unsigned() } else { @@ -248,14 +276,24 @@ unsafe fn mm_extract_epi64(val: __m128i, idx: i32) -> u64 { } } +/// Updates a CRC-32C state with one 64-bit word. +/// +/// # Safety +/// +/// The current CPU must support SSE4.2. #[inline] #[target_feature(enable = "sse4.2")] unsafe fn mm_crc32c_u64(crc: u32, val: u64) -> u32 { - // SAFETY: SSE4.2 intrinsics are available via this function's #[target_feature] attribute. - _mm_crc32_u64(crc as u64, val) as u32 + let result = _mm_crc32_u64(u64::from(crc), val); + let [b0, b1, b2, b3, ..] = result.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) } -// x^n mod P (iSCSI / CRC32C), in O(log n) time. +/// Computes x^n modulo the reflected CRC-32C polynomial in O(log n) time. +/// +/// # Safety +/// +/// The current CPU must support SSE4.2 and PCLMULQDQ. #[target_feature(enable = "sse4.2,pclmulqdq")] unsafe fn xnmodp_iscsi_sse(mut n: u64) -> u32 { // SAFETY: SSE4.2/PCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -266,7 +304,7 @@ unsafe fn xnmodp_iscsi_sse(mut n: u64) -> u32 { let mut low: u32; while n > 191 { - stack = (stack << 1) + (n & 1); + stack = (stack << 1) | (n & 1); n = (n >> 1).strict_sub(16); } stack = !stack; @@ -293,6 +331,12 @@ unsafe fn xnmodp_iscsi_sse(mut n: u64) -> u32 { } } +/// Computes the PCLMULQDQ coefficient that shifts `crc` by `nbytes`. +/// +/// # Safety +/// +/// The current CPU must support SSE4.2 and PCLMULQDQ, and `nbytes` must be at +/// least five so the x-power exponent is nonnegative. #[inline] #[target_feature(enable = "sse4.2,pclmulqdq")] unsafe fn crc_shift_iscsi_sse(crc: u32, nbytes: usize) -> __m128i { @@ -304,6 +348,31 @@ unsafe fn crc_shift_iscsi_sse(crc: u32, nbytes: usize) -> __m128i { } } +/// Loads 16 bytes without requiring alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 16 initialized bytes. The source may be +/// unaligned because the bytes are copied into aligned local storage. +#[inline] +unsafe fn load_unaligned_128(ptr: *const u8) -> __m128i { + let mut value = core::mem::MaybeUninit::<__m128i>::uninit(); + + // SAFETY: The caller guarantees a readable 16-byte source. `value` is an + // aligned, non-overlapping 16-byte destination, and every bit pattern is + // valid for an integer vector. + unsafe { + core::ptr::copy_nonoverlapping(ptr, value.as_mut_ptr().cast::(), 16); + value.assume_init() + } +} + +/// Updates CRC-32C with the SSE4.2/PCLMULQDQ v4s3x3 fusion kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE4.2 and PCLMULQDQ. `buf` must point to +/// `len` initialized readable bytes in one allocation. #[inline] #[target_feature(enable = "sse4.2,pclmulqdq")] unsafe fn crc32c_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -319,22 +388,22 @@ unsafe fn crc32c_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: us // Handle 8-byte alignment. if (buf as usize & 8) != 0 && len >= 8 { - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::())); buf = buf.add(8); len = len.strict_sub(8); } if len >= 144 { - let blk = (len.strict_sub(8)) / 136; + let blk = len.strict_sub(8).strict_div(136); let klen = blk.strict_mul(24); let buf2_start = buf; let mut crc1 = 0u32; let mut crc2 = 0u32; - let mut x0 = _mm_loadu_si128(buf2_start as *const __m128i); - let mut x1 = _mm_loadu_si128(buf2_start.add(16) as *const __m128i); - let mut x2 = _mm_loadu_si128(buf2_start.add(32) as *const __m128i); - let mut x3 = _mm_loadu_si128(buf2_start.add(48) as *const __m128i); + let mut x0 = load_unaligned_128(buf2_start); + let mut x1 = load_unaligned_128(buf2_start.add(16)); + let mut x2 = load_unaligned_128(buf2_start.add(32)); + let mut x3 = load_unaligned_128(buf2_start.add(48)); // iSCSI-specific folding constant. let mut k = _mm_setr_epi32(0x740eef02u32.cast_signed(), 0, 0x9e4addf8u32.cast_signed(), 0); @@ -357,26 +426,29 @@ unsafe fn crc32c_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: us let mut y3 = clmul_lo_sse(x3, k); x3 = clmul_hi_sse(x3, k); - y0 = _mm_xor_si128(y0, _mm_loadu_si128(buf2 as *const __m128i)); + y0 = _mm_xor_si128(y0, load_unaligned_128(buf2)); x0 = _mm_xor_si128(x0, y0); - y1 = _mm_xor_si128(y1, _mm_loadu_si128(buf2.add(16) as *const __m128i)); + y1 = _mm_xor_si128(y1, load_unaligned_128(buf2.add(16))); x1 = _mm_xor_si128(x1, y1); - y2 = _mm_xor_si128(y2, _mm_loadu_si128(buf2.add(32) as *const __m128i)); + y2 = _mm_xor_si128(y2, load_unaligned_128(buf2.add(32))); x2 = _mm_xor_si128(x2, y2); - y3 = _mm_xor_si128(y3, _mm_loadu_si128(buf2.add(48) as *const __m128i)); + y3 = _mm_xor_si128(y3, load_unaligned_128(buf2.add(48))); x3 = _mm_xor_si128(x3, y3); - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = mm_crc32c_u64(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen + 8) as *const u64)); - crc2 = mm_crc32c_u64(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2) + 8) as *const u64)); - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(16) as *const u64)); - crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen + 16) as *const u64)); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::())); + crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen).cast::())); + crc2 = mm_crc32c_u64(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)).cast::())); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(8).cast::())); + crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen.strict_add(8)).cast::())); + crc2 = mm_crc32c_u64( + crc2, + ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(8)).cast::()), + ); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(16).cast::())); + crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen.strict_add(16)).cast::())); crc2 = mm_crc32c_u64( crc2, - ptr::read_unaligned(buf.add(klen.strict_mul(2) + 16) as *const u64), + ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(16)).cast::()), ); buf = buf.add(24); @@ -405,29 +477,35 @@ unsafe fn crc32c_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: us x0 = _mm_xor_si128(x0, y0); // Final scalar chunk. - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = mm_crc32c_u64(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen + 8) as *const u64)); - crc2 = mm_crc32c_u64(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2) + 8) as *const u64)); - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(16) as *const u64)); - crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen + 16) as *const u64)); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::())); + crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen).cast::())); + crc2 = mm_crc32c_u64(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)).cast::())); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(8).cast::())); + crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen.strict_add(8)).cast::())); crc2 = mm_crc32c_u64( crc2, - ptr::read_unaligned(buf.add(klen.strict_mul(2) + 16) as *const u64), + ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(8)).cast::()), + ); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(16).cast::())); + crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen.strict_add(16)).cast::())); + crc2 = mm_crc32c_u64( + crc2, + ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(16)).cast::()), ); buf = buf.add(24); - let vc0 = crc_shift_iscsi_sse(crc0, klen.strict_mul(2) + 8); - let vc1 = crc_shift_iscsi_sse(crc1, klen + 8); + let vc0 = crc_shift_iscsi_sse(crc0, klen.strict_mul(2).strict_add(8)); + let vc1 = crc_shift_iscsi_sse(crc1, klen.strict_add(8)); let mut vc = mm_extract_epi64(_mm_xor_si128(vc0, vc1), 0); // Reduce 128 bits to 32 bits, and multiply by x^32. let x0_low = mm_extract_epi64(x0, 0); let x0_high = mm_extract_epi64(x0, 1); let x0_combined = mm_extract_epi64( - crc_shift_iscsi_sse(mm_crc32c_u64(mm_crc32c_u64(0, x0_low), x0_high), klen.strict_mul(3) + 8), + crc_shift_iscsi_sse( + mm_crc32c_u64(mm_crc32c_u64(0, x0_low), x0_high), + klen.strict_mul(3).strict_add(8), + ), 0, ); vc ^= x0_combined; @@ -435,13 +513,13 @@ unsafe fn crc32c_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: us // Final 8 bytes. buf = buf.add(klen.strict_mul(2)); crc0 = crc2; - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf as *const u64) ^ vc); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::()) ^ vc); buf = buf.add(8); len = len.strict_sub(8); } while len >= 8 { - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::())); buf = buf.add(8); len = len.strict_sub(8); } @@ -458,7 +536,7 @@ unsafe fn crc32c_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: us /// Safe wrapper for CRC-32C fusion kernel (SSE4.2 + PCLMULQDQ). #[inline] -pub fn crc32c_iscsi_sse_v4s3x3_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_sse_v4s3x3_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSE4.2 + PCLMULQDQ before selecting this kernel. unsafe { crc32c_iscsi_sse_v4s3x3(crc, data.as_ptr(), data.len()) } } @@ -515,41 +593,72 @@ fn crc32c_fusion_nway(crc: u32, data: &[u8], update: fn(u32, &[u } #[inline] -pub fn crc32c_iscsi_sse_v4s3x3_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_sse_v4s3x3_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<2>(crc, data, crc32c_iscsi_sse_v4s3x3_safe) } #[inline] -pub fn crc32c_iscsi_sse_v4s3x3_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_sse_v4s3x3_4way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<4>(crc, data, crc32c_iscsi_sse_v4s3x3_safe) } #[inline] -pub fn crc32c_iscsi_sse_v4s3x3_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_sse_v4s3x3_7way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<7>(crc, data, crc32c_iscsi_sse_v4s3x3_safe) } #[inline] -pub fn crc32c_iscsi_sse_v4s3x3_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_sse_v4s3x3_8way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<8>(crc, data, crc32c_iscsi_sse_v4s3x3_safe) } +/// Multiplies the low 64-bit lanes of each 128-bit element over GF(2). +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul_lo_avx512_vpclmulqdq(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_clmulepi64_epi128(a, b, 0) } +/// Multiplies the high 64-bit lanes of each 128-bit element over GF(2). +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul_hi_avx512_vpclmulqdq(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_clmulepi64_epi128(a, b, 17) } +/// Loads 64 bytes without requiring alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 64 initialized bytes. The source may be +/// unaligned because the bytes are copied into aligned local storage. +#[inline] +unsafe fn load_unaligned_512(ptr: *const u8) -> __m512i { + let mut value = core::mem::MaybeUninit::<__m512i>::uninit(); + + // SAFETY: The caller guarantees a readable 64-byte source. `value` is an + // aligned, non-overlapping 64-byte destination, and every bit pattern is + // valid for an integer vector. + unsafe { + core::ptr::copy_nonoverlapping(ptr, value.as_mut_ptr().cast::(), 64); + value.assume_init() + } +} + +/// Updates CRC-32C with the AVX-512/VPCLMULQDQ v3x2 fusion kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE4.2, AVX-512F/VL/BW/DQ, and VPCLMULQDQ. +/// `buf` must point to `len` initialized readable bytes in one allocation. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,sse4.2")] unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -565,16 +674,16 @@ unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, // Align to 64-byte boundary (cache line). while (buf as usize & 56) != 0 && len >= 8 { - crc0 = _mm_crc32_u64(crc0 as u64, ptr::read_unaligned(buf as *const u64)) as u32; + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::())); buf = buf.add(8); len = len.strict_sub(8); } if len >= 384 { // Load three 512-bit vectors (192 bytes). - let mut x0 = _mm512_loadu_si512(buf as *const __m512i); - let mut x1 = _mm512_loadu_si512(buf.add(64) as *const __m512i); - let mut x2 = _mm512_loadu_si512(buf.add(128) as *const __m512i); + let mut x0 = load_unaligned_512(buf); + let mut x1 = load_unaligned_512(buf.add(64)); + let mut x2 = load_unaligned_512(buf.add(128)); // Broadcast folding constant to each 128-bit lane. let k_128 = _mm_setr_epi32(0xa87ab8a8u32.cast_signed(), 0, 0xab7aff2au32.cast_signed(), 0); @@ -592,9 +701,9 @@ unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, let mut y2 = clmul_lo_avx512_vpclmulqdq(x2, k); x2 = clmul_hi_avx512_vpclmulqdq(x2, k); - x0 = _mm512_ternarylogic_epi64(x0, y0, _mm512_loadu_si512(buf.add(192) as *const __m512i), 0x96); - x1 = _mm512_ternarylogic_epi64(x1, y1, _mm512_loadu_si512(buf.add(256) as *const __m512i), 0x96); - x2 = _mm512_ternarylogic_epi64(x2, y2, _mm512_loadu_si512(buf.add(320) as *const __m512i), 0x96); + x0 = _mm512_ternarylogic_epi64(x0, y0, load_unaligned_512(buf.add(192)), 0x96); + x1 = _mm512_ternarylogic_epi64(x1, y1, load_unaligned_512(buf.add(256)), 0x96); + x2 = _mm512_ternarylogic_epi64(x2, y2, load_unaligned_512(buf.add(320)), 0x96); buf = buf.add(384); len = len.strict_sub(384); @@ -608,9 +717,9 @@ unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, y2 = clmul_lo_avx512_vpclmulqdq(x2, k); x2 = clmul_hi_avx512_vpclmulqdq(x2, k); - x0 = _mm512_ternarylogic_epi64(x0, y0, _mm512_loadu_si512(buf as *const __m512i), 0x96); - x1 = _mm512_ternarylogic_epi64(x1, y1, _mm512_loadu_si512(buf.add(64) as *const __m512i), 0x96); - x2 = _mm512_ternarylogic_epi64(x2, y2, _mm512_loadu_si512(buf.add(128) as *const __m512i), 0x96); + x0 = _mm512_ternarylogic_epi64(x0, y0, load_unaligned_512(buf), 0x96); + x1 = _mm512_ternarylogic_epi64(x1, y1, load_unaligned_512(buf.add(64)), 0x96); + x2 = _mm512_ternarylogic_epi64(x2, y2, load_unaligned_512(buf.add(128)), 0x96); // Second folding step. y0 = clmul_lo_avx512_vpclmulqdq(x0, k); @@ -620,9 +729,9 @@ unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, y2 = clmul_lo_avx512_vpclmulqdq(x2, k); x2 = clmul_hi_avx512_vpclmulqdq(x2, k); - x0 = _mm512_ternarylogic_epi64(x0, y0, _mm512_loadu_si512(buf.add(192) as *const __m512i), 0x96); - x1 = _mm512_ternarylogic_epi64(x1, y1, _mm512_loadu_si512(buf.add(256) as *const __m512i), 0x96); - x2 = _mm512_ternarylogic_epi64(x2, y2, _mm512_loadu_si512(buf.add(320) as *const __m512i), 0x96); + x0 = _mm512_ternarylogic_epi64(x0, y0, load_unaligned_512(buf.add(192)), 0x96); + x1 = _mm512_ternarylogic_epi64(x1, y1, load_unaligned_512(buf.add(256)), 0x96); + x2 = _mm512_ternarylogic_epi64(x2, y2, load_unaligned_512(buf.add(320)), 0x96); buf = buf.add(384); len = len.strict_sub(384); @@ -673,12 +782,12 @@ unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, let mut z0 = _mm_ternarylogic_epi64(lane0, lane1, lane2, 0x96); z0 = _mm_xor_si128(z0, lane3); - crc0 = _mm_crc32_u64(0, mm_extract_epi64(z0, 0)) as u32; - crc0 = _mm_crc32_u64(crc0 as u64, mm_extract_epi64(z0, 1)) as u32; + crc0 = mm_crc32c_u64(0, mm_extract_epi64(z0, 0)); + crc0 = mm_crc32c_u64(crc0, mm_extract_epi64(z0, 1)); } while len >= 8 { - crc0 = _mm_crc32_u64(crc0 as u64, ptr::read_unaligned(buf as *const u64)) as u32; + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::())); buf = buf.add(8); len = len.strict_sub(8); } @@ -695,28 +804,28 @@ unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, /// Safe wrapper for CRC-32C fusion kernel (AVX-512 VPCLMULQDQ v3x2). #[inline] -pub fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies AVX-512 VPCLMULQDQ before selecting this kernel. unsafe { crc32c_iscsi_avx512_vpclmulqdq_v3x2(crc, data.as_ptr(), data.len()) } } #[inline] -pub fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<2>(crc, data, crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe) } #[inline] -pub fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_4way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<4>(crc, data, crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe) } #[inline] -pub fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_7way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<7>(crc, data, crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe) } #[inline] -pub fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_8way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<8>(crc, data, crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe) } @@ -790,7 +899,8 @@ const fn reduce128_crc32(hi: u64, lo: u64, poly: u32) -> u32 { bit = bit.strict_sub(1); } - val as u32 + let [b0, b1, b2, b3, ..] = val.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) } /// Compute x^n mod (x^32 + poly) in GF(2) (poly is the normal CRC polynomial without the x^32 @@ -922,45 +1032,68 @@ impl BitXorAssign for Simd128 { } impl Simd128 { + /// Creates a vector from its high and low 64-bit lanes. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn new(high: u64, low: u64) -> Self { - // SAFETY: SSE2 intrinsics are available via this function's #[target_feature] attribute. Self(_mm_set_epi64x(high.cast_signed(), low.cast_signed())) } + /// Shifts the vector right by eight bytes, filling the high bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn shift_right_8(self) -> Self { - // SAFETY: SSE2 intrinsics are available via this function's #[target_feature] attribute. Self(_mm_srli_si128::<8>(self.0)) } + /// Shifts the vector left by 12 bytes, filling the low bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn shift_left_12(self) -> Self { - // SAFETY: SSE2 intrinsics are available via this function's #[target_feature] attribute. Self(_mm_slli_si128::<12>(self.0)) } + /// Computes the bitwise AND of two vectors. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn and(self, mask: Self) -> Self { - // SAFETY: SSE2 intrinsics are available via this function's #[target_feature] attribute. Self(_mm_and_si128(self.0, mask.0)) } - /// Fold 16 bytes using the reflected CRC32 folding primitive. + /// Folds one reflected 16-byte lane and XORs the supplied input lane. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_16(self, coeff: Self, data_to_xor: Self) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. let h = _mm_clmulepi64_si128::<0x10>(self.0, coeff.0); let l = _mm_clmulepi64_si128::<0x01>(self.0, coeff.0); Self(_mm_xor_si128(_mm_xor_si128(h, l), data_to_xor.0)) } - /// Fold 16 bytes down to CRC32 width (reflected mode). + /// Folds a reflected CRC state from 128 bits to the width-32 reduction state. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_width_crc32_reflected(self, high: u64, low: u64) -> Self { @@ -985,7 +1118,11 @@ impl Simd128 { } } - /// Barrett reduction for reflected CRC32; returns the updated (pre-inverted) CRC. + /// Applies reflected Barrett reduction and returns the width-32 CRC state. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn barrett_crc32_reflected(self, poly: u64, mu: u64) -> u32 { @@ -997,11 +1134,16 @@ impl Simd128 { let xorred = _mm_xor_si128(self.0, clmul2); let hi = _mm_srli_si128::<8>(xorred); - _mm_cvtsi128_si64(hi) as u32 + _mm_cvtsi128_si32(hi).cast_unsigned() } } } +/// Combines eight folded lanes and applies width-32 Barrett reduction. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn finalize_lanes_crc32_ieee_reflected(x: [Simd128; 8]) -> u32 { @@ -1025,6 +1167,11 @@ unsafe fn finalize_lanes_crc32_ieee_reflected(x: [Simd128; 8]) -> u32 { } } +/// Folds one or more 128-byte blocks into a reflected CRC-32 state. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_crc32_ieee_reflected(state: u32, first: &[Simd128; 8], rest: &[[Simd128; 8]]) -> u32 { @@ -1091,7 +1238,7 @@ pub(crate) unsafe fn crc32_ieee_pclmul(crc: u32, data: &[u8]) -> u32 { /// Safe wrapper for CRC-32 (IEEE) PCLMUL kernel. #[inline] -pub fn crc32_ieee_pclmul_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_pclmul_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_pclmul(crc, data) } } @@ -1122,13 +1269,13 @@ unsafe fn crc32_ieee_pclmul_small(crc: u32, data: &[u8]) -> u32 { let coeff_16b = Simd128::new(keys[2], keys[1]); // Load the first lane and inject the initial CRC state (low 32 bits). - let mut x0 = Simd128(_mm_loadu_si128(buf as *const __m128i)); + let mut x0 = Simd128(load_unaligned_128(buf)); x0 ^= Simd128::new(0, crc as u64); buf = buf.add(16); len = len.strict_sub(16); while len >= 16 { - let chunk = Simd128(_mm_loadu_si128(buf as *const __m128i)); + let chunk = Simd128(load_unaligned_128(buf)); x0 = x0.fold_16(coeff_16b, chunk); buf = buf.add(16); len = len.strict_sub(16); @@ -1144,13 +1291,18 @@ unsafe fn crc32_ieee_pclmul_small(crc: u32, data: &[u8]) -> u32 { /// Safe wrapper for CRC-32 (IEEE) "pclmul-small" kernel. #[inline] -pub fn crc32_ieee_pclmul_small_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_pclmul_small_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies PCLMULQDQ (directly or via VPCLMUL-ready) before selecting this path. unsafe { crc32_ieee_pclmul_small(crc, data) } } // CRC-32 (IEEE) Multi-stream Folding (SSSE3 + PCLMULQDQ) +/// Folds one 128-byte block into eight PCLMULQDQ lanes. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_block_128_crc32(x: &mut [Simd128; 8], chunk: &[Simd128; 8], coeff: Simd128) { @@ -1176,6 +1328,12 @@ unsafe fn fold_block_128_crc32(x: &mut [Simd128; 8], chunk: &[Simd128; 8], coeff } } +/// Folds 128-byte blocks through two PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. `blocks` must +/// contain at least two blocks. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_crc32_ieee_2way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1199,7 +1357,7 @@ unsafe fn update_simd_crc32_ieee_2way(state: u32, blocks: &[[Simd128; 8]]) -> u3 const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); // Double-unrolled loop while i.strict_add(DOUBLE_GROUP) <= aligned { @@ -1247,6 +1405,11 @@ unsafe fn update_simd_crc32_ieee_2way(state: u32, blocks: &[[Simd128; 8]]) -> u3 } } +/// Folds 128-byte blocks through four PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_crc32_ieee_4way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1278,7 +1441,7 @@ unsafe fn update_simd_crc32_ieee_4way(state: u32, blocks: &[[Simd128; 8]]) -> u3 const DOUBLE_GROUP: usize = 8; // 2 × 4-way = 8 blocks = 1KB let mut i: usize = 4; - let double_aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let double_aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); // Double-unrolled loop while i.strict_add(DOUBLE_GROUP) <= double_aligned { @@ -1304,7 +1467,7 @@ unsafe fn update_simd_crc32_ieee_4way(state: u32, blocks: &[[Simd128; 8]]) -> u3 } // Handle remaining 4-block groups - let aligned = (blocks.len() / 4) * 4; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(4)); while i < aligned { fold_block_128_crc32(&mut s0, &blocks[i], coeff_512b); fold_block_128_crc32(&mut s1, &blocks[i.strict_add(1)], coeff_512b); @@ -1328,6 +1491,11 @@ unsafe fn update_simd_crc32_ieee_4way(state: u32, blocks: &[[Simd128; 8]]) -> u3 } } +/// Folds 128-byte blocks through seven PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_crc32_ieee_7way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1341,7 +1509,7 @@ unsafe fn update_simd_crc32_ieee_7way(state: u32, blocks: &[[Simd128; 8]]) -> u3 return update_simd_crc32_ieee_reflected(state, first, rest); } - let aligned = (blocks.len() / 7) * 7; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(7)); let coeff_896b = Simd128::new(CRC32_IEEE_STREAM.fold_896b.0, CRC32_IEEE_STREAM.fold_896b.1); let coeff_128b = Simd128::new(CRC32_IEEE_KEYS_REFLECTED[4], CRC32_IEEE_KEYS_REFLECTED[3]); @@ -1403,6 +1571,11 @@ unsafe fn update_simd_crc32_ieee_7way(state: u32, blocks: &[[Simd128; 8]]) -> u3 } } +/// Folds 128-byte blocks through eight PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_crc32_ieee_8way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1416,7 +1589,7 @@ unsafe fn update_simd_crc32_ieee_8way(state: u32, blocks: &[[Simd128; 8]]) -> u3 return update_simd_crc32_ieee_reflected(state, first, rest); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(8)); let coeff_1024b = Simd128::new(CRC32_IEEE_STREAM.fold_1024b.0, CRC32_IEEE_STREAM.fold_1024b.1); let coeff_128b = Simd128::new(CRC32_IEEE_KEYS_REFLECTED[4], CRC32_IEEE_KEYS_REFLECTED[3]); @@ -1483,6 +1656,10 @@ unsafe fn update_simd_crc32_ieee_8way(state: u32, blocks: &[[Simd128; 8]]) -> u3 } /// CRC-32 (IEEE / ISO-HDLC) update using PCLMULQDQ folding (2-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] pub(crate) unsafe fn crc32_ieee_pclmul_2way(crc: u32, data: &[u8]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1504,12 +1681,16 @@ pub(crate) unsafe fn crc32_ieee_pclmul_2way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_pclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_pclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_pclmul_2way(crc, data) } } /// CRC-32 (IEEE / ISO-HDLC) update using PCLMULQDQ folding (4-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] pub(crate) unsafe fn crc32_ieee_pclmul_4way(crc: u32, data: &[u8]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1527,12 +1708,16 @@ pub(crate) unsafe fn crc32_ieee_pclmul_4way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_pclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_pclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_pclmul_4way(crc, data) } } /// CRC-32 (IEEE / ISO-HDLC) update using PCLMULQDQ folding (7-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] pub(crate) unsafe fn crc32_ieee_pclmul_7way(crc: u32, data: &[u8]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1550,12 +1735,16 @@ pub(crate) unsafe fn crc32_ieee_pclmul_7way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_pclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_pclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_pclmul_7way(crc, data) } } /// CRC-32 (IEEE / ISO-HDLC) update using PCLMULQDQ folding (8-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] pub(crate) unsafe fn crc32_ieee_pclmul_8way(crc: u32, data: &[u8]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1573,27 +1762,38 @@ pub(crate) unsafe fn crc32_ieee_pclmul_8way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_pclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_pclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_pclmul_8way(crc, data) } } +/// Multiplies the high lane of each 128-bit element in `a` by the low lane in `b`. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul10_vpclmul(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_clmulepi64_epi128(a, b, 0x10) } +/// Multiplies the low lane of each 128-bit element in `a` by the high lane in `b`. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul01_vpclmul(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_clmulepi64_epi128(a, b, 0x01) } +/// Folds four reflected 16-byte lanes and XORs their supplied input lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn fold_16_crc32_reflected_vpclmul(state: __m512i, coeff: __m512i, data: __m512i) -> __m512i { @@ -1602,6 +1802,12 @@ unsafe fn fold_16_crc32_reflected_vpclmul(state: __m512i, coeff: __m512i, data: unsafe { _mm512_ternarylogic_epi64(clmul10_vpclmul(state, coeff), clmul01_vpclmul(state, coeff), data, 0x96) } } +/// Folds a leading 128-byte block and the remaining blocks with VPCLMULQDQ. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_crc32_ieee_reflected_vpclmul(state: u32, first: &[Simd128; 8], rest: &[[Simd128; 8]]) -> u32 { @@ -1625,8 +1831,8 @@ unsafe fn update_simd_crc32_ieee_reflected_vpclmul(state: u32, first: &[Simd128; // Load the first 128-byte block as 2×512-bit registers (8×16B lanes). let base = first.as_ptr().cast::(); - let mut x0 = _mm512_loadu_si512(base.cast::<__m512i>()); - let mut x1 = _mm512_loadu_si512(base.add(64).cast::<__m512i>()); + let mut x0 = load_unaligned_512(base); + let mut x1 = load_unaligned_512(base.add(64)); // Inject CRC into the first lane (low 32 bits). let injected = _mm512_setr_epi64(state as i64, 0, 0, 0, 0, 0, 0, 0); @@ -1634,8 +1840,8 @@ unsafe fn update_simd_crc32_ieee_reflected_vpclmul(state: u32, first: &[Simd128; for block in rest { let ptr = block.as_ptr().cast::(); - let y0 = _mm512_loadu_si512(ptr.cast::<__m512i>()); - let y1 = _mm512_loadu_si512(ptr.add(64).cast::<__m512i>()); + let y0 = load_unaligned_512(ptr); + let y1 = load_unaligned_512(ptr.add(64)); x0 = fold_16_crc32_reflected_vpclmul(x0, coeff, y0); x1 = fold_16_crc32_reflected_vpclmul(x1, coeff, y1); } @@ -1695,18 +1901,21 @@ pub(crate) unsafe fn crc32_ieee_vpclmul(crc: u32, data: &[u8]) -> u32 { /// Safe wrapper for CRC-32 (IEEE) VPCLMUL kernel. #[inline] -pub fn crc32_ieee_vpclmul_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpclmul_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies AVX-512 VPCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_vpclmul(crc, data) } } // CRC-32 (IEEE) Multi-stream Folding (AVX-512 VPCLMULQDQ) +/// Broadcasts one pair of folding coefficients across four 128-bit lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. // Each 128-bit lane expects: low64 = pair.1, high64 = pair.0. _mm512_setr_epi64( pair.1.cast_signed(), @@ -1720,6 +1929,11 @@ unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { ) } +/// Folds four reflected lanes without injecting another data vector. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn fold_only_crc32_reflected_vpclmul(state: __m512i, coeff: __m512i) -> __m512i { @@ -1728,6 +1942,11 @@ unsafe fn fold_only_crc32_reflected_vpclmul(state: __m512i, coeff: __m512i) -> _ unsafe { _mm512_xor_si512(clmul10_vpclmul(state, coeff), clmul01_vpclmul(state, coeff)) } } +/// Loads one 128-byte block as two unaligned 512-bit vectors. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn load_128b_block(block: &[Simd128; 8]) -> (__m512i, __m512i) { @@ -1735,12 +1954,18 @@ unsafe fn load_128b_block(block: &[Simd128; 8]) -> (__m512i, __m512i) { // attribute. Pointer arithmetic: block is 128 bytes, so ptr+64 is within bounds. unsafe { let ptr = block.as_ptr().cast::(); - let y0 = _mm512_loadu_si512(ptr.cast::<__m512i>()); - let y1 = _mm512_loadu_si512(ptr.add(64).cast::<__m512i>()); + let y0 = load_unaligned_512(ptr); + let y1 = load_unaligned_512(ptr.add(64)); (y0, y1) } } +/// Combines two four-lane VPCLMULQDQ states and applies CRC-32 reduction. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ, VPCLMULQDQ, SSE2, and +/// PCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,sse2,pclmulqdq")] unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i) -> u32 { @@ -1771,6 +1996,13 @@ unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i) -> u32 { } } +/// Folds 128-byte blocks through two VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. +/// `blocks` must contain at least two blocks. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_crc32_ieee_vpclmul_2way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -1843,6 +2075,12 @@ unsafe fn update_simd_crc32_ieee_vpclmul_2way(state: u32, blocks: &[[Simd128; 8] } } +/// Folds 128-byte blocks through four VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_crc32_ieee_vpclmul_4way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -1957,6 +2195,12 @@ unsafe fn update_simd_crc32_ieee_vpclmul_4way(state: u32, blocks: &[[Simd128; 8] } } +/// Folds 128-byte blocks through seven VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_crc32_ieee_vpclmul_7way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -2056,6 +2300,12 @@ unsafe fn update_simd_crc32_ieee_vpclmul_7way(state: u32, blocks: &[[Simd128; 8] } } +/// Folds 128-byte blocks through eight VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_crc32_ieee_vpclmul_8way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -2164,6 +2414,11 @@ unsafe fn update_simd_crc32_ieee_vpclmul_8way(state: u32, blocks: &[[Simd128; 8] } /// CRC-32 (IEEE / ISO-HDLC) update using AVX-512 VPCLMULQDQ folding (2-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] pub(crate) unsafe fn crc32_ieee_vpclmul_2way(crc: u32, data: &[u8]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -2181,12 +2436,17 @@ pub(crate) unsafe fn crc32_ieee_vpclmul_2way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vpclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies AVX-512 VPCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_vpclmul_2way(crc, data) } } /// CRC-32 (IEEE / ISO-HDLC) update using AVX-512 VPCLMULQDQ folding (4-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] pub(crate) unsafe fn crc32_ieee_vpclmul_4way(crc: u32, data: &[u8]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -2204,12 +2464,17 @@ pub(crate) unsafe fn crc32_ieee_vpclmul_4way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vpclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies AVX-512 VPCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_vpclmul_4way(crc, data) } } /// CRC-32 (IEEE / ISO-HDLC) update using AVX-512 VPCLMULQDQ folding (7-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] pub(crate) unsafe fn crc32_ieee_vpclmul_7way(crc: u32, data: &[u8]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -2227,12 +2492,17 @@ pub(crate) unsafe fn crc32_ieee_vpclmul_7way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vpclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies AVX-512 VPCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_vpclmul_7way(crc, data) } } /// CRC-32 (IEEE / ISO-HDLC) update using AVX-512 VPCLMULQDQ folding (8-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] pub(crate) unsafe fn crc32_ieee_vpclmul_8way(crc: u32, data: &[u8]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -2250,7 +2520,7 @@ pub(crate) unsafe fn crc32_ieee_vpclmul_8way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vpclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies AVX-512 VPCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_vpclmul_8way(crc, data) } } @@ -2263,6 +2533,14 @@ mod tests { use super::*; + fn deterministic_data(len: usize, multiplier: u8, increment: u8) -> Vec { + (0u8..=u8::MAX) + .cycle() + .take(len) + .map(|byte| byte.wrapping_mul(multiplier).wrapping_add(increment)) + .collect() + } + fn vpclmul_available_for_tests() -> bool { !cfg!(miri) && std::arch::is_x86_feature_detected!("avx512f") @@ -2298,17 +2576,21 @@ mod tests { return; } - for len in [ - 0usize, 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 255, 256, 1024, - ] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(31).wrapping_add(7)); + let data = deterministic_data(4096usize.strict_add(15), 31, 7); + for &init in &[!0u32, 0, 0x1234_5678, 0xDEAD_BEEF] { + for &offset in &[0usize, 1, 3, 7, 15] { + for &len in &[ + 0usize, 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 129, 255, 256, 1024, 4096, + ] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32_slice16_ieee(init, input); + assert_eq!( + crc32_ieee_pclmul_safe(init, input), + expected, + "init={init:#x} offset={offset} len={len}" + ); + } } - - let portable = super::super::portable::crc32_slice16_ieee(!0, &data) ^ !0; - let pclmul = crc32_ieee_pclmul_safe(!0, &data) ^ !0; - assert_eq!(pclmul, portable, "len={len}"); } } @@ -2318,15 +2600,21 @@ mod tests { return; } - for len in [0usize, 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(31).wrapping_add(7)); - } + let data_len = 127usize.strict_add(15); + let data = deterministic_data(data_len, 31, 7); - let portable = super::super::portable::crc32_slice16_ieee(!0, &data) ^ !0; - let pclmul_small = crc32_ieee_pclmul_small_safe(!0, &data) ^ !0; - assert_eq!(pclmul_small, portable, "len={len}"); + for &init in &[!0u32, 0, 0x1234_5678, 0xDEAD_BEEF] { + for &offset in &[0usize, 1, 3, 7, 15] { + for &len in &[0usize, 1, 2, 3, 4, 7, 8, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32_slice16_ieee(init, input); + assert_eq!( + crc32_ieee_pclmul_small_safe(init, input), + expected, + "init={init:#x} offset={offset} len={len}" + ); + } + } } } @@ -2336,17 +2624,21 @@ mod tests { return; } - for len in [ - 0usize, 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 255, 256, 1024, 4096, - ] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(31).wrapping_add(7)); + let data = deterministic_data(4096usize.strict_add(15), 31, 7); + for &init in &[!0u32, 0, 0x1234_5678, 0xDEAD_BEEF] { + for &offset in &[0usize, 1, 3, 7, 15] { + for &len in &[ + 0usize, 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 129, 255, 256, 1024, 4096, + ] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32_slice16_ieee(init, input); + assert_eq!( + crc32_ieee_vpclmul_safe(init, input), + expected, + "init={init:#x} offset={offset} len={len}" + ); + } } - - let portable = super::super::portable::crc32_slice16_ieee(!0, &data) ^ !0; - let vpclmul = crc32_ieee_vpclmul_safe(!0, &data) ^ !0; - assert_eq!(vpclmul, portable, "len={len}"); } } @@ -2356,34 +2648,36 @@ mod tests { return; } - for &init in &[!0u32, 0x1234_5678, 0xDEAD_BEEF] { - for len in [0usize, 1, 15, 16, 127, 128, 255, 256, 1024, 4096, 16384] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(31).wrapping_add(7)); + let data = deterministic_data(16384usize.strict_add(15), 31, 7); + for &init in &[!0u32, 0, 0x1234_5678, 0xDEAD_BEEF] { + for &offset in &[0usize, 1, 3, 7, 15] { + for &len in &[ + 0usize, 1, 15, 16, 127, 128, 129, 255, 256, 257, 511, 512, 513, 895, 896, 897, 1023, 1024, 1025, 1791, 1792, + 1793, 2047, 2048, 2049, 4095, 4096, 4097, 16384, + ] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32_slice16_ieee(init, input); + assert_eq!( + crc32_ieee_pclmul_2way_safe(init, input), + expected, + "2way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32_ieee_pclmul_4way_safe(init, input), + expected, + "4way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32_ieee_pclmul_7way_safe(init, input), + expected, + "7way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32_ieee_pclmul_8way_safe(init, input), + expected, + "8way init={init:#x} offset={offset} len={len}" + ); } - - let expected = super::super::portable::crc32_slice16_ieee(init, &data); - assert_eq!( - crc32_ieee_pclmul_2way_safe(init, &data), - expected, - "2way init={init:#x} len={len}" - ); - assert_eq!( - crc32_ieee_pclmul_4way_safe(init, &data), - expected, - "4way init={init:#x} len={len}" - ); - assert_eq!( - crc32_ieee_pclmul_7way_safe(init, &data), - expected, - "7way init={init:#x} len={len}" - ); - assert_eq!( - crc32_ieee_pclmul_8way_safe(init, &data), - expected, - "8way init={init:#x} len={len}" - ); } } } @@ -2394,34 +2688,63 @@ mod tests { return; } - for &init in &[!0u32, 0x1234_5678, 0xDEAD_BEEF] { - for len in [0usize, 1, 15, 16, 127, 128, 255, 256, 1024, 4096, 16384] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(31).wrapping_add(7)); + let data = deterministic_data(16384usize.strict_add(15), 31, 7); + for &init in &[!0u32, 0, 0x1234_5678, 0xDEAD_BEEF] { + for &offset in &[0usize, 1, 3, 7, 15] { + for &len in &[ + 0usize, 1, 15, 16, 127, 128, 129, 255, 256, 257, 511, 512, 513, 895, 896, 897, 1023, 1024, 1025, 1791, 1792, + 1793, 2047, 2048, 2049, 4095, 4096, 4097, 16384, + ] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32_slice16_ieee(init, input); + assert_eq!( + crc32_ieee_vpclmul_2way_safe(init, input), + expected, + "2way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32_ieee_vpclmul_4way_safe(init, input), + expected, + "4way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32_ieee_vpclmul_7way_safe(init, input), + expected, + "7way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32_ieee_vpclmul_8way_safe(init, input), + expected, + "8way init={init:#x} offset={offset} len={len}" + ); } + } + } + } - let expected = super::super::portable::crc32_slice16_ieee(init, &data); - assert_eq!( - crc32_ieee_vpclmul_2way_safe(init, &data), - expected, - "2way init={init:#x} len={len}" - ); - assert_eq!( - crc32_ieee_vpclmul_4way_safe(init, &data), - expected, - "4way init={init:#x} len={len}" - ); - assert_eq!( - crc32_ieee_vpclmul_7way_safe(init, &data), - expected, - "7way init={init:#x} len={len}" - ); - assert_eq!( - crc32_ieee_vpclmul_8way_safe(init, &data), - expected, - "8way init={init:#x} len={len}" - ); + #[test] + fn test_crc32c_sse42_matches_portable() { + if !std::arch::is_x86_feature_detected!("sse4.2") { + return; + } + + let data: Vec = (0u8..=u8::MAX) + .cycle() + .take(4096usize.strict_add(7)) + .map(|byte| byte.wrapping_mul(17).wrapping_add(3)) + .collect(); + + for &init in &[!0u32, 0, 0x0123_4567, 0x89AB_CDEF] { + for &offset in &[0usize, 1, 3, 7] { + for &len in &[0usize, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 16, 31, 32, 127, 128, 4096] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32c_slice16(init, input); + assert_eq!( + crc32c_sse42_safe(init, input), + expected, + "init={init:#x} offset={offset} len={len}" + ); + } } } } @@ -2432,34 +2755,38 @@ mod tests { return; } - for &init in &[!0u32, 0x0123_4567, 0x89AB_CDEF] { - for len in [0usize, 1, 7, 8, 15, 16, 127, 128, 255, 256, 2048, 8192] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(17).wrapping_add(3)); + for &init in &[!0u32, 0, 0x0123_4567, 0x89AB_CDEF] { + for len in [ + 0usize, 1, 7, 8, 15, 16, 127, 128, 255, 256, 511, 512, 513, 1023, 1024, 1025, 1791, 1792, 1793, 2047, 2048, + 2049, 4095, 4096, 4097, 8192, + ] { + for &offset in &[0usize, 1, 3, 7] { + let data_len = offset.strict_add(len); + let data = deterministic_data(data_len, 17, 3); + + let input = &data[offset..]; + let expected = super::super::portable::crc32c_slice16(init, input); + assert_eq!( + crc32c_sse42_2way_safe(init, input), + expected, + "2way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_sse42_4way_safe(init, input), + expected, + "4way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_sse42_7way_safe(init, input), + expected, + "7way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_sse42_8way_safe(init, input), + expected, + "8way init={init:#x} offset={offset} len={len}" + ); } - - let expected = super::super::portable::crc32c_slice16(init, &data); - assert_eq!( - crc32c_sse42_2way_safe(init, &data), - expected, - "2way init={init:#x} len={len}" - ); - assert_eq!( - crc32c_sse42_4way_safe(init, &data), - expected, - "4way init={init:#x} len={len}" - ); - assert_eq!( - crc32c_sse42_7way_safe(init, &data), - expected, - "7way init={init:#x} len={len}" - ); - assert_eq!( - crc32c_sse42_8way_safe(init, &data), - expected, - "8way init={init:#x} len={len}" - ); } } } @@ -2470,34 +2797,86 @@ mod tests { return; } - for &init in &[!0u32, 0x0123_4567, 0x89AB_CDEF] { - for len in [0usize, 1, 7, 8, 15, 16, 127, 128, 255, 256, 2048, 8192, 65536] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(19).wrapping_add(5)); + let data = deterministic_data(65536usize.strict_add(7), 19, 5); + for &init in &[!0u32, 0, 0x0123_4567, 0x89AB_CDEF] { + for &offset in &[0usize, 1, 3, 7] { + for &len in &[ + 0usize, 1, 7, 8, 15, 16, 63, 64, 127, 128, 135, 136, 143, 144, 145, 255, 256, 271, 272, 273, 383, 384, 385, + 511, 512, 513, 1023, 1024, 1025, 2048, 8192, 65536, + ] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32c_slice16(init, input); + assert_eq!( + crc32c_iscsi_sse_v4s3x3_safe(init, input), + expected, + "fusion/1way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_sse_v4s3x3_2way_safe(init, input), + expected, + "fusion/2way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_sse_v4s3x3_4way_safe(init, input), + expected, + "fusion/4way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_sse_v4s3x3_7way_safe(init, input), + expected, + "fusion/7way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_sse_v4s3x3_8way_safe(init, input), + expected, + "fusion/8way init={init:#x} offset={offset} len={len}" + ); } + } + } + } - let expected = super::super::portable::crc32c_slice16(init, &data); - assert_eq!( - crc32c_iscsi_sse_v4s3x3_2way_safe(init, &data), - expected, - "fusion/2way init={init:#x} len={len}" - ); - assert_eq!( - crc32c_iscsi_sse_v4s3x3_4way_safe(init, &data), - expected, - "fusion/4way init={init:#x} len={len}" - ); - assert_eq!( - crc32c_iscsi_sse_v4s3x3_7way_safe(init, &data), - expected, - "fusion/7way init={init:#x} len={len}" - ); - assert_eq!( - crc32c_iscsi_sse_v4s3x3_8way_safe(init, &data), - expected, - "fusion/8way init={init:#x} len={len}" - ); + #[test] + fn test_crc32c_fusion_vpclmul_multistream_matches_portable() { + if !(vpclmul_available_for_tests() && std::arch::is_x86_feature_detected!("sse4.2")) { + return; + } + + let data = deterministic_data(8192usize.strict_add(7), 19, 5); + for &init in &[!0u32, 0, 0x0123_4567, 0x89AB_CDEF] { + for &offset in &[0usize, 1, 3, 7] { + for &len in &[ + 0usize, 1, 7, 8, 15, 16, 63, 64, 127, 128, 191, 192, 383, 384, 385, 511, 512, 513, 1023, 1024, 1025, 2048, + 8192, + ] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32c_slice16(init, input); + assert_eq!( + crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe(init, input), + expected, + "fusion-vpclmul/1way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_avx512_vpclmulqdq_v3x2_2way_safe(init, input), + expected, + "fusion-vpclmul/2way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_avx512_vpclmulqdq_v3x2_4way_safe(init, input), + expected, + "fusion-vpclmul/4way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_avx512_vpclmulqdq_v3x2_7way_safe(init, input), + expected, + "fusion-vpclmul/7way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_avx512_vpclmulqdq_v3x2_8way_safe(init, input), + expected, + "fusion-vpclmul/8way init={init:#x} offset={offset} len={len}" + ); + } } } } diff --git a/src/checksum/crc64/aarch64.rs b/src/checksum/crc64/aarch64.rs index 9e9a0c25..94f9cd49 100644 --- a/src/checksum/crc64/aarch64.rs +++ b/src/checksum/crc64/aarch64.rs @@ -5,13 +5,8 @@ //! //! # Safety //! -//! Uses `unsafe` for ARM SIMD intrinsics. Callers must ensure PMULL is -//! available before executing the accelerated path (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] -// This module is intrinsics-heavy; unsafe blocks are per-function with SAFETY justifications. +//! Uses `unsafe` for ARM SIMD intrinsics. Callers must establish NEON, AES +//! (PMULL), and SHA3 (EOR3) support as required by each accelerated path. use core::{ arch::aarch64::*, @@ -24,50 +19,71 @@ use crate::checksum::common::clmul::{CRC64_NVME_STREAM, CRC64_XZ_STREAM, Crc64Cl #[derive(Copy, Clone, Debug)] struct Simd(uint8x16_t); -#[allow(non_camel_case_types)] -type poly64_t = u64; - impl Simd { + /// Multiplies two carry-less 64-bit polynomials into one 128-bit vector. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES (PMULL). #[inline] #[target_feature(enable = "neon", enable = "aes")] - unsafe fn from_mul(a: poly64_t, b: poly64_t) -> Self { - // SAFETY: Caller guarantees NEON+AES (PMULL) is available. Intrinsics operate on registers. + unsafe fn from_mul(a: u64, b: u64) -> Self { let mul = vmull_p64(a, b); Self(vreinterpretq_u8_p128(mul)) } + /// Returns the vector's two 64-bit polynomial lanes in low-to-high order. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES. #[inline] #[target_feature(enable = "neon", enable = "aes")] - unsafe fn into_poly64s(self) -> [poly64_t; 2] { - // SAFETY: Caller guarantees NEON+AES is available. Intrinsics operate on registers. + unsafe fn into_poly64s(self) -> [u64; 2] { let x = vreinterpretq_p64_u8(self.0); [vgetq_lane_p64(x, 0), vgetq_lane_p64(x, 1)] } + /// Returns the high 64-bit polynomial lane. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES. #[inline] #[target_feature(enable = "neon", enable = "aes")] - unsafe fn high_64(self) -> poly64_t { - // SAFETY: Caller guarantees NEON+AES is available. Intrinsics operate on registers. + unsafe fn high_64(self) -> u64 { let x = vreinterpretq_p64_u8(self.0); vgetq_lane_p64(x, 1) } + /// Returns the low 64-bit polynomial lane. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES. #[inline] #[target_feature(enable = "neon", enable = "aes")] - unsafe fn low_64(self) -> poly64_t { - // SAFETY: Caller guarantees NEON+AES is available. Intrinsics operate on registers. + unsafe fn low_64(self) -> u64 { let x = vreinterpretq_p64_u8(self.0); vgetq_lane_p64(x, 0) } + /// Creates a vector from its high and low 64-bit lanes. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn new(high: u64, low: u64) -> Self { - // SAFETY: Caller guarantees NEON is available. Intrinsics operate on registers. Self(vcombine_u8(vcreate_u8(low), vcreate_u8(high))) } /// Fold 16 bytes: `(coeff.low ⊗ self.low) ⊕ (coeff.high ⊗ self.high)`. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES (PMULL). #[inline] #[target_feature(enable = "neon", enable = "aes")] unsafe fn fold_16(self, coeff: Self) -> Self { @@ -86,9 +102,13 @@ impl Simd { /// /// Equivalent to `self.fold_16(Simd::new(high, low))` but avoids repeatedly /// extracting the coefficient lanes inside hot loops. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES (PMULL). #[inline] #[target_feature(enable = "neon", enable = "aes")] - unsafe fn fold_16_pair(self, coeff_low: poly64_t, coeff_high: poly64_t) -> Self { + unsafe fn fold_16_pair(self, coeff_low: u64, coeff_high: u64) -> Self { // SAFETY: Caller guarantees NEON+AES (PMULL) is available. All operations are register // computations. unsafe { @@ -100,6 +120,10 @@ impl Simd { } /// Fold 8 bytes: `self.high ⊕ (coeff ⊗ self.low)`. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES (PMULL). #[inline] #[target_feature(enable = "neon", enable = "aes")] unsafe fn fold_8(self, coeff: u64) -> Self { @@ -114,6 +138,10 @@ impl Simd { } /// Barrett reduction to finalize the CRC. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES (PMULL). #[inline] #[target_feature(enable = "neon", enable = "aes")] unsafe fn barrett(self, poly: u64, mu: u64) -> u64 { @@ -145,6 +173,11 @@ impl BitXorAssign for Simd { } } +/// Folds a sequence of 128-byte blocks and reduces it to a CRC-64 state. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: &Crc64ClmulConstants) -> u64 { // SAFETY: Caller guarantees NEON+AES (PMULL) target features are available (dispatch check). @@ -167,6 +200,11 @@ unsafe fn update_simd(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: } // unsafe } +/// Folds a sequence of 128-byte blocks with EOR3 and reduces it to a CRC-64 state. +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_eor3(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: &Crc64ClmulConstants) -> u64 { // SAFETY: Caller guarantees NEON+AES+SHA3 (PMULL+EOR3) target features are available (dispatch @@ -188,6 +226,11 @@ unsafe fn update_simd_eor3(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], co } // unsafe } +/// Reduces eight folded SIMD lanes to a CRC-64 state. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[inline(always)] unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { // SAFETY: Caller guarantees NEON+AES (PMULL) target features are available (dispatch check). @@ -218,6 +261,11 @@ unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { // PMULL multi-stream (2-way/3-way, 128B blocks) +/// Folds one 128-byte block into the current SIMD state. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff_low: u64, coeff_high: u64) { @@ -239,6 +287,10 @@ unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff_low: u64, c /// /// This reduces the XOR dependency chain: instead of `chunk ^ (h ^ l)` (2 XORs), /// we use `veor3(chunk, h, l)` (1 instruction). +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[inline] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn fold_block_128_eor3(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff_low: u64, coeff_high: u64) { @@ -260,9 +312,13 @@ unsafe fn fold_block_128_eor3(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff_low: u /// /// Computes: `data ^ pmull(coeff_low, x.low) ^ pmull(coeff_high, x.high)` /// using a single 3-way XOR instruction. +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[inline] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] -unsafe fn fold_lane_eor3(x: Simd, data: Simd, coeff_low: poly64_t, coeff_high: poly64_t) -> Simd { +unsafe fn fold_lane_eor3(x: Simd, data: Simd, coeff_low: u64, coeff_high: u64) -> Simd { // SAFETY: Caller guarantees NEON+AES+SHA3 (PMULL+EOR3) is available. All operations are register // computations. unsafe { @@ -274,6 +330,11 @@ unsafe fn fold_lane_eor3(x: Simd, data: Simd, coeff_low: poly64_t, coeff_high: p } } +/// Folds 128-byte blocks through two independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_2way( state: u64, @@ -304,7 +365,7 @@ unsafe fn update_simd_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP); // Double-unrolled loop while i.strict_add(DOUBLE_GROUP) <= aligned { @@ -353,6 +414,11 @@ unsafe fn update_simd_2way( } // unsafe } +/// Folds 128-byte blocks through three independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_3way( state: u64, @@ -392,7 +458,7 @@ unsafe fn update_simd_3way( const DOUBLE_GROUP: usize = 6; // 2 × 3-way = 6 blocks = 768B let mut i: usize = 3; - let double_aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let double_aligned = blocks.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP); // Double-unrolled loop while i.strict_add(DOUBLE_GROUP) <= double_aligned { @@ -416,7 +482,7 @@ unsafe fn update_simd_3way( } // Handle remaining 3-block groups - let aligned = (blocks.len() / 3) * 3; + let aligned = blocks.len().strict_div(3).strict_mul(3); while i < aligned { fold_block_128(&mut s0, &blocks[i], coeff_384_low, coeff_384_high); fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_384_low, coeff_384_high); @@ -459,6 +525,10 @@ unsafe fn update_simd_3way( /// /// Maintains two independent fold streams and uses EOR3 when merging each /// polynomial product with input data. +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_eor3_2way( state: u64, @@ -490,7 +560,7 @@ unsafe fn update_simd_eor3_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP); // Double-unrolled loop while i.strict_add(DOUBLE_GROUP) <= aligned { @@ -546,6 +616,10 @@ unsafe fn update_simd_eor3_2way( /// /// Maintains three independent fold streams and uses EOR3 when merging each /// polynomial product with input data. +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_eor3_3way( state: u64, @@ -586,7 +660,7 @@ unsafe fn update_simd_eor3_3way( const DOUBLE_GROUP: usize = 6; // 2 × 3-way = 6 blocks = 768B let mut i: usize = 3; - let double_aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let double_aligned = blocks.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP); // Double-unrolled loop while i.strict_add(DOUBLE_GROUP) <= double_aligned { @@ -610,7 +684,7 @@ unsafe fn update_simd_eor3_3way( } // Handle remaining 3-block groups - let aligned = (blocks.len() / 3) * 3; + let aligned = blocks.len().strict_div(3).strict_mul(3); while i < aligned { fold_block_128_eor3(&mut s0, &blocks[i], coeff_384_low, coeff_384_high); fold_block_128_eor3(&mut s1, &blocks[i.strict_add(1)], coeff_384_low, coeff_384_high); @@ -650,6 +724,11 @@ unsafe fn update_simd_eor3_3way( } // unsafe } +/// Computes CRC-64 with two EOR3-assisted PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn crc64_pmull_eor3_2way( mut state: u64, @@ -671,6 +750,11 @@ unsafe fn crc64_pmull_eor3_2way( } } +/// Computes CRC-64 with three EOR3-assisted PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn crc64_pmull_eor3_3way( mut state: u64, @@ -693,6 +777,11 @@ unsafe fn crc64_pmull_eor3_3way( } } +/// Computes CRC-64 with two independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc64_pmull_2way( mut state: u64, @@ -714,6 +803,11 @@ unsafe fn crc64_pmull_2way( } } +/// Computes CRC-64 with three independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc64_pmull_3way( mut state: u64, @@ -736,6 +830,11 @@ unsafe fn crc64_pmull_3way( } } +/// Computes CRC-64 with PMULL folding. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc64_pmull(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 8]) -> u64 { // SAFETY: Caller guarantees NEON+AES (PMULL) is available. align_to produces valid sub-slices. @@ -754,6 +853,10 @@ unsafe fn crc64_pmull(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants /// PMULL+EOR3 path: uses EOR3 to combine three XOR operands. /// /// Available on ARMv8.2+ with SHA3 extension (Apple M1+, AWS Graviton3+). +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn crc64_pmull_eor3( mut state: u64, @@ -778,6 +881,10 @@ unsafe fn crc64_pmull_eor3( /// /// This avoids constructing the full 128-byte fold state when only complete /// 16-byte lanes are available. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc64_pmull_small( mut state: u64, @@ -864,6 +971,7 @@ pub(crate) unsafe fn crc64_xz_pmull_small(crc: u64, data: &[u8]) -> u64 { /// /// Requires PMULL (crypto/aes). Caller must verify via /// `crate::platform::caps().has(aarch64::SVE2_PMULL)` and `PMULL_READY` before selecting. +#[cfg(any(feature = "std", test))] #[target_feature(enable = "aes", enable = "neon")] pub(crate) unsafe fn crc64_xz_sve2_pmull_2way(crc: u64, data: &[u8]) -> u64 { // SAFETY: the caller guarantees `SVE2_PMULL` and `PMULL_READY` before selecting this kernel. @@ -879,6 +987,7 @@ pub(crate) unsafe fn crc64_xz_sve2_pmull_2way(crc: u64, data: &[u8]) -> u64 { /// /// Requires PMULL (crypto/aes). Caller must verify via /// `crate::platform::caps().has(aarch64::SVE2_PMULL)` and `PMULL_READY` before selecting. +#[cfg(any(feature = "std", test))] #[target_feature(enable = "aes", enable = "neon")] pub(crate) unsafe fn crc64_xz_sve2_pmull_3way(crc: u64, data: &[u8]) -> u64 { // SAFETY: the caller guarantees `SVE2_PMULL` and `PMULL_READY` before selecting this kernel. @@ -934,6 +1043,7 @@ pub(crate) unsafe fn crc64_nvme_pmull_small(crc: u64, data: &[u8]) -> u64 { /// /// Requires PMULL (crypto/aes). Caller must verify via /// `crate::platform::caps().has(aarch64::SVE2_PMULL)` and `PMULL_READY` before selecting. +#[cfg(any(feature = "std", test))] #[target_feature(enable = "aes", enable = "neon")] pub(crate) unsafe fn crc64_nvme_sve2_pmull_2way(crc: u64, data: &[u8]) -> u64 { // SAFETY: the caller guarantees `SVE2_PMULL` and `PMULL_READY` before selecting this kernel. @@ -949,6 +1059,7 @@ pub(crate) unsafe fn crc64_nvme_sve2_pmull_2way(crc: u64, data: &[u8]) -> u64 { /// /// Requires PMULL (crypto/aes). Caller must verify via /// `crate::platform::caps().has(aarch64::SVE2_PMULL)` and `PMULL_READY` before selecting. +#[cfg(any(feature = "std", test))] #[target_feature(enable = "aes", enable = "neon")] pub(crate) unsafe fn crc64_nvme_sve2_pmull_3way(crc: u64, data: &[u8]) -> u64 { // SAFETY: the caller guarantees `SVE2_PMULL` and `PMULL_READY` before selecting this kernel. @@ -1103,122 +1214,130 @@ pub(crate) unsafe fn crc64_nvme_pmull_eor3_3way(crc: u64, data: &[u8]) -> u64 { /// Safe wrapper for CRC-64-XZ PMULL kernel. #[inline] -pub fn crc64_xz_pmull_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_xz_pmull(crc, data) } } /// Safe wrapper for CRC-64-XZ PMULL small-buffer kernel. #[inline] -pub fn crc64_xz_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_xz_pmull_small(crc, data) } } /// Safe wrapper for CRC-64-XZ tuned "SVE2 PMULL" tier (single-stream). #[inline] -pub fn crc64_xz_sve2_pmull_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(feature = "std")] +pub(super) fn crc64_xz_sve2_pmull_safe(crc: u64, data: &[u8]) -> u64 { crc64_xz_pmull_safe(crc, data) } /// Safe wrapper for CRC-64-XZ tuned "SVE2 PMULL" tier (small-buffer). #[inline] -pub fn crc64_xz_sve2_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(feature = "std")] +pub(super) fn crc64_xz_sve2_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { crc64_xz_pmull_small_safe(crc, data) } /// Safe wrapper for CRC-64-XZ tuned SVE2 2-way PMULL kernel. #[inline] -pub fn crc64_xz_sve2_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc64_xz_sve2_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies SVE2_PMULL + PMULL before selecting this kernel. unsafe { crc64_xz_sve2_pmull_2way(crc, data) } } /// Safe wrapper for CRC-64-XZ tuned SVE2 3-way PMULL kernel. #[inline] -pub fn crc64_xz_sve2_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc64_xz_sve2_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies SVE2_PMULL + PMULL before selecting this kernel. unsafe { crc64_xz_sve2_pmull_3way(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL kernel. #[inline] -pub fn crc64_nvme_pmull_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_nvme_pmull(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL small-buffer kernel. #[inline] -pub fn crc64_nvme_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_nvme_pmull_small(crc, data) } } /// Safe wrapper for CRC-64-NVME tuned "SVE2 PMULL" tier (single-stream). #[inline] -pub fn crc64_nvme_sve2_pmull_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(feature = "std")] +pub(super) fn crc64_nvme_sve2_pmull_safe(crc: u64, data: &[u8]) -> u64 { crc64_nvme_pmull_safe(crc, data) } /// Safe wrapper for CRC-64-NVME tuned "SVE2 PMULL" tier (small-buffer). #[inline] -pub fn crc64_nvme_sve2_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(feature = "std")] +pub(super) fn crc64_nvme_sve2_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { crc64_nvme_pmull_small_safe(crc, data) } /// Safe wrapper for CRC-64-NVME tuned SVE2 2-way PMULL kernel. #[inline] -pub fn crc64_nvme_sve2_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc64_nvme_sve2_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies SVE2_PMULL + PMULL before selecting this kernel. unsafe { crc64_nvme_sve2_pmull_2way(crc, data) } } /// Safe wrapper for CRC-64-NVME tuned SVE2 3-way PMULL kernel. #[inline] -pub fn crc64_nvme_sve2_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc64_nvme_sve2_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies SVE2_PMULL + PMULL before selecting this kernel. unsafe { crc64_nvme_sve2_pmull_3way(crc, data) } } /// Safe wrapper for CRC-64-XZ PMULL+EOR3 kernel. #[inline] -pub fn crc64_xz_pmull_eor3_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_eor3_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL_EOR3_READY before selecting this kernel. unsafe { crc64_xz_pmull_eor3(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL+EOR3 kernel. #[inline] -pub fn crc64_nvme_pmull_eor3_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_eor3_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL_EOR3_READY before selecting this kernel. unsafe { crc64_nvme_pmull_eor3(crc, data) } } /// Safe wrapper for CRC-64-XZ PMULL+EOR3 2-way kernel. #[inline] -pub fn crc64_xz_pmull_eor3_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_eor3_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL_EOR3_READY before selecting this kernel. unsafe { crc64_xz_pmull_eor3_2way(crc, data) } } /// Safe wrapper for CRC-64-XZ PMULL+EOR3 3-way kernel. #[inline] -pub fn crc64_xz_pmull_eor3_3way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_eor3_3way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL_EOR3_READY before selecting this kernel. unsafe { crc64_xz_pmull_eor3_3way(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL+EOR3 2-way kernel. #[inline] -pub fn crc64_nvme_pmull_eor3_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_eor3_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL_EOR3_READY before selecting this kernel. unsafe { crc64_nvme_pmull_eor3_2way(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL+EOR3 3-way kernel. #[inline] -pub fn crc64_nvme_pmull_eor3_3way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_eor3_3way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL_EOR3_READY before selecting this kernel. unsafe { crc64_nvme_pmull_eor3_3way(crc, data) } } @@ -1317,28 +1436,28 @@ pub(crate) unsafe fn crc64_nvme_pmull_3way(crc: u64, data: &[u8]) -> u64 { /// Safe wrapper for CRC-64-XZ PMULL 2-way kernel. #[inline] -pub fn crc64_xz_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_xz_pmull_2way(crc, data) } } /// Safe wrapper for CRC-64-XZ PMULL 3-way kernel. #[inline] -pub fn crc64_xz_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_xz_pmull_3way(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL 2-way kernel. #[inline] -pub fn crc64_nvme_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_nvme_pmull_2way(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL 3-way kernel. #[inline] -pub fn crc64_nvme_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_nvme_pmull_3way(crc, data) } } @@ -1356,7 +1475,11 @@ mod tests { fn make_data(len: usize) -> alloc::vec::Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(17).wrapping_add((i >> 3) as u8)) + .map(|i| { + let [low, high, ..] = i.to_le_bytes(); + let shifted = low.strict_shr(3) | high.strict_shl(5); + low.wrapping_mul(17).wrapping_add(shifted) + }) .collect() } @@ -1588,7 +1711,6 @@ mod tests { return; } - // SAFETY: We just checked AES+SHA3 is available. let crc = crc64_xz_pmull_eor3_safe(!0, TEST_DATA) ^ !0; assert_eq!(crc, 0x995D_C9BB_DF19_39FA); } @@ -1599,7 +1721,6 @@ mod tests { return; } - // SAFETY: We just checked AES+SHA3 is available. let crc = crc64_nvme_pmull_eor3_safe(!0, TEST_DATA) ^ !0; assert_eq!(crc, 0xAE8B_1486_0A79_9888); } diff --git a/src/checksum/crc64/config.rs b/src/checksum/crc64/config.rs index 3fa7e4a3..c685a664 100644 --- a/src/checksum/crc64/config.rs +++ b/src/checksum/crc64/config.rs @@ -35,10 +35,6 @@ pub enum Crc64Force { Vpmsum, /// Force s390x VGFM (if supported). Vgfm, - /// Force riscv64 Zbc carryless multiply (if supported). - Zbc, - /// Force riscv64 Zvbc (vector carryless multiply) folding (if supported). - Zvbc, } impl Crc64Force { @@ -56,8 +52,6 @@ impl Crc64Force { Self::Sve2Pmull => "sve2-pmull", Self::Vpmsum => "vpmsum", Self::Vgfm => "vgfm", - Self::Zbc => "zbc", - Self::Zvbc => "zvbc", } } } @@ -111,20 +105,20 @@ fn parse_force_env() -> Crc64Force { if value.eq_ignore_ascii_case("vgfm") || value.eq_ignore_ascii_case("gfmsum") { return Crc64Force::Vgfm; } - if value.eq_ignore_ascii_case("zbc") { - return Crc64Force::Zbc; - } - if value.eq_ignore_ascii_case("zvbc") || value.eq_ignore_ascii_case("vclmul") { - return Crc64Force::Zvbc; - } - Crc64Force::Auto } #[inline] #[must_use] -#[allow(unused_variables)] // `caps` only used on x86_64/aarch64 fn clamp_force_to_caps(requested: Crc64Force, caps: Caps) -> Crc64Force { + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "x86_64" + )))] + let _ = caps; + match requested { Crc64Force::Auto | Crc64Force::Reference | Crc64Force::Portable => requested, Crc64Force::Pclmul => { @@ -191,24 +185,6 @@ fn clamp_force_to_caps(requested: Crc64Force, caps: Caps) -> Crc64Force { } Crc64Force::Auto } - Crc64Force::Zbc => { - #[cfg(target_arch = "riscv64")] - { - if caps.has(crate::platform::caps::riscv::ZBC) { - return Crc64Force::Zbc; - } - } - Crc64Force::Auto - } - Crc64Force::Zvbc => { - #[cfg(target_arch = "riscv64")] - { - if caps.has(crate::platform::caps::riscv::ZVBC) { - return Crc64Force::Zvbc; - } - } - Crc64Force::Auto - } } } @@ -249,7 +225,7 @@ fn config(caps: Caps) -> Crc64Config { /// platform capabilities. #[inline] #[must_use] -pub fn get() -> Crc64Config { +pub(super) fn get() -> Crc64Config { #[cfg(feature = "std")] { use std::sync::OnceLock; diff --git a/src/checksum/crc64/kernels.rs b/src/checksum/crc64/kernels.rs index 317b5cd8..3890646e 100644 --- a/src/checksum/crc64/kernels.rs +++ b/src/checksum/crc64/kernels.rs @@ -20,23 +20,23 @@ //! - CRC64-specific dispatch functions generated via macro /// Portable fallback kernel name. -pub use kernels::PORTABLE_SLICE16 as PORTABLE; +pub(in crate::checksum) use kernels::PORTABLE_SLICE16 as PORTABLE; /// Reference (bitwise) kernel name - always available for force mode. -pub use kernels::REFERENCE; +pub(in crate::checksum) use kernels::REFERENCE; use crate::checksum::common::kernels; // Kernel Name Tables (per architecture) #[cfg(target_arch = "x86_64")] -pub mod x86_64 { +pub(in crate::checksum) mod x86_64 { use super::super::x86_64 as arch; use crate::checksum::dispatchers::Crc64Fn; // CRC64-XZ Kernel Function Arrays /// XZ PCLMUL kernels: [1-way, 2-way, 4-way, 7-way, 8-way] - pub const XZ_PCLMUL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_PCLMUL: [Crc64Fn; 5] = [ arch::crc64_xz_pclmul_safe, arch::crc64_xz_pclmul_2way_safe, arch::crc64_xz_pclmul_4way_safe, @@ -44,10 +44,10 @@ pub mod x86_64 { arch::crc64_xz_pclmul_8way_safe, ]; /// XZ PCLMUL small buffer kernel. - pub const XZ_PCLMUL_SMALL: Crc64Fn = arch::crc64_xz_pclmul_small_safe; + pub(in crate::checksum) const XZ_PCLMUL_SMALL: Crc64Fn = arch::crc64_xz_pclmul_small_safe; /// XZ VPCLMUL kernels: [1-way, 2-way, 4-way, 7-way, 8-way] - pub const XZ_VPCLMUL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_VPCLMUL: [Crc64Fn; 5] = [ arch::crc64_xz_vpclmul_safe, arch::crc64_xz_vpclmul_2way_safe, arch::crc64_xz_vpclmul_4way_safe, @@ -55,12 +55,12 @@ pub mod x86_64 { arch::crc64_xz_vpclmul_8way_safe, ]; /// XZ VPCLMUL 4×512-bit kernel. - pub const XZ_VPCLMUL_4X512: Crc64Fn = arch::crc64_xz_vpclmul_4x512_safe; + pub(in crate::checksum) const XZ_VPCLMUL_4X512: Crc64Fn = arch::crc64_xz_vpclmul_4x512_safe; // CRC64-NVME Kernel Function Arrays /// NVME PCLMUL kernels: [1-way, 2-way, 4-way, 7-way, 8-way] - pub const NVME_PCLMUL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_PCLMUL: [Crc64Fn; 5] = [ arch::crc64_nvme_pclmul_safe, arch::crc64_nvme_pclmul_2way_safe, arch::crc64_nvme_pclmul_4way_safe, @@ -68,10 +68,10 @@ pub mod x86_64 { arch::crc64_nvme_pclmul_8way_safe, ]; /// NVME PCLMUL small buffer kernel. - pub const NVME_PCLMUL_SMALL: Crc64Fn = arch::crc64_nvme_pclmul_small_safe; + pub(in crate::checksum) const NVME_PCLMUL_SMALL: Crc64Fn = arch::crc64_nvme_pclmul_small_safe; /// NVME VPCLMUL kernels: [1-way, 2-way, 4-way, 7-way, 8-way] - pub const NVME_VPCLMUL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_VPCLMUL: [Crc64Fn; 5] = [ arch::crc64_nvme_vpclmul_safe, arch::crc64_nvme_vpclmul_2way_safe, arch::crc64_nvme_vpclmul_4way_safe, @@ -81,9 +81,7 @@ pub mod x86_64 { } #[cfg(target_arch = "aarch64")] -pub mod aarch64 { - #![cfg_attr(not(any(test, feature = "std")), allow(dead_code))] - #![cfg_attr(not(any(target_os = "linux", target_os = "android")), allow(dead_code))] +pub(in crate::checksum) mod aarch64 { use super::super::aarch64 as arch; use crate::checksum::dispatchers::Crc64Fn; @@ -92,7 +90,7 @@ pub mod aarch64 { // Note: aarch64 only supports up to 3-way, slots 3-4 are duplicates /// XZ PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)] - pub const XZ_PMULL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_PMULL: [Crc64Fn; 5] = [ arch::crc64_xz_pmull_safe, arch::crc64_xz_pmull_2way_safe, arch::crc64_xz_pmull_3way_safe, @@ -100,10 +98,10 @@ pub mod aarch64 { arch::crc64_xz_pmull_3way_safe, // dup for index consistency ]; /// XZ PMULL small buffer kernel. - pub const XZ_PMULL_SMALL: Crc64Fn = arch::crc64_xz_pmull_small_safe; + pub(in crate::checksum) const XZ_PMULL_SMALL: Crc64Fn = arch::crc64_xz_pmull_small_safe; /// XZ PMULL+EOR3 kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)] - pub const XZ_PMULL_EOR3: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_PMULL_EOR3: [Crc64Fn; 5] = [ arch::crc64_xz_pmull_eor3_safe, arch::crc64_xz_pmull_eor3_2way_safe, arch::crc64_xz_pmull_eor3_3way_safe, @@ -111,21 +109,23 @@ pub mod aarch64 { arch::crc64_xz_pmull_eor3_3way_safe, // dup for index consistency ]; + #[cfg(feature = "std")] /// XZ SVE2 PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)] - pub const XZ_SVE2_PMULL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_SVE2_PMULL: [Crc64Fn; 5] = [ arch::crc64_xz_sve2_pmull_safe, arch::crc64_xz_sve2_pmull_2way_safe, arch::crc64_xz_sve2_pmull_3way_safe, arch::crc64_xz_sve2_pmull_3way_safe, // dup for index consistency arch::crc64_xz_sve2_pmull_3way_safe, // dup for index consistency ]; + #[cfg(feature = "std")] /// XZ SVE2 PMULL small buffer kernel. - pub const XZ_SVE2_PMULL_SMALL: Crc64Fn = arch::crc64_xz_sve2_pmull_small_safe; + pub(in crate::checksum) const XZ_SVE2_PMULL_SMALL: Crc64Fn = arch::crc64_xz_sve2_pmull_small_safe; // CRC64-NVME Kernel Function Arrays /// NVME PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)] - pub const NVME_PMULL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_PMULL: [Crc64Fn; 5] = [ arch::crc64_nvme_pmull_safe, arch::crc64_nvme_pmull_2way_safe, arch::crc64_nvme_pmull_3way_safe, @@ -133,10 +133,10 @@ pub mod aarch64 { arch::crc64_nvme_pmull_3way_safe, // dup for index consistency ]; /// NVME PMULL small buffer kernel. - pub const NVME_PMULL_SMALL: Crc64Fn = arch::crc64_nvme_pmull_small_safe; + pub(in crate::checksum) const NVME_PMULL_SMALL: Crc64Fn = arch::crc64_nvme_pmull_small_safe; /// NVME PMULL+EOR3 kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)] - pub const NVME_PMULL_EOR3: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_PMULL_EOR3: [Crc64Fn; 5] = [ arch::crc64_nvme_pmull_eor3_safe, arch::crc64_nvme_pmull_eor3_2way_safe, arch::crc64_nvme_pmull_eor3_3way_safe, @@ -144,27 +144,29 @@ pub mod aarch64 { arch::crc64_nvme_pmull_eor3_3way_safe, // dup for index consistency ]; + #[cfg(feature = "std")] /// NVME SVE2 PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)] - pub const NVME_SVE2_PMULL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_SVE2_PMULL: [Crc64Fn; 5] = [ arch::crc64_nvme_sve2_pmull_safe, arch::crc64_nvme_sve2_pmull_2way_safe, arch::crc64_nvme_sve2_pmull_3way_safe, arch::crc64_nvme_sve2_pmull_3way_safe, // dup for index consistency arch::crc64_nvme_sve2_pmull_3way_safe, // dup for index consistency ]; + #[cfg(feature = "std")] /// NVME SVE2 PMULL small buffer kernel. - pub const NVME_SVE2_PMULL_SMALL: Crc64Fn = arch::crc64_nvme_sve2_pmull_small_safe; + pub(in crate::checksum) const NVME_SVE2_PMULL_SMALL: Crc64Fn = arch::crc64_nvme_sve2_pmull_small_safe; } #[cfg(target_arch = "powerpc64")] -pub mod power { +pub(in crate::checksum) mod power { use super::super::power as arch; use crate::checksum::dispatchers::Crc64Fn; // CRC64-XZ Kernel Function Arrays /// XZ VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)] - pub const XZ_VPMSUM: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_VPMSUM: [Crc64Fn; 5] = [ arch::crc64_xz_vpmsum_safe, arch::crc64_xz_vpmsum_2way_safe, arch::crc64_xz_vpmsum_4way_safe, @@ -175,7 +177,7 @@ pub mod power { // CRC64-NVME Kernel Function Arrays /// NVME VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)] - pub const NVME_VPMSUM: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_VPMSUM: [Crc64Fn; 5] = [ arch::crc64_nvme_vpmsum_safe, arch::crc64_nvme_vpmsum_2way_safe, arch::crc64_nvme_vpmsum_4way_safe, @@ -185,7 +187,7 @@ pub mod power { } #[cfg(target_arch = "s390x")] -pub mod s390x { +pub(in crate::checksum) mod s390x { use super::super::s390x as arch; use crate::checksum::dispatchers::Crc64Fn; @@ -193,7 +195,7 @@ pub mod s390x { // Note: s390x only supports up to 4-way, slots 3-4 are duplicates /// XZ VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)] - pub const XZ_VGFM: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_VGFM: [Crc64Fn; 5] = [ arch::crc64_xz_vgfm_safe, arch::crc64_xz_vgfm_2way_safe, arch::crc64_xz_vgfm_4way_safe, @@ -204,7 +206,7 @@ pub mod s390x { // CRC64-NVME Kernel Function Arrays /// NVME VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)] - pub const NVME_VGFM: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_VGFM: [Crc64Fn; 5] = [ arch::crc64_nvme_vgfm_safe, arch::crc64_nvme_vgfm_2way_safe, arch::crc64_nvme_vgfm_4way_safe, @@ -212,69 +214,3 @@ pub mod s390x { arch::crc64_nvme_vgfm_4way_safe, // dup for index consistency ]; } - -#[cfg(target_arch = "riscv64")] -#[allow(dead_code)] -pub mod riscv64 { - use super::super::riscv64 as arch; - use crate::checksum::dispatchers::Crc64Fn; - - /// Zbc kernel names: [1-way, 2-way, 4-way, 8-way, 8-way(dup)] - pub const ZBC_NAMES: &[&str] = &[ - "riscv64/zbc", - "riscv64/zbc-2way", - "riscv64/zbc-4way", - "riscv64/zbc-8way", - "riscv64/zbc-8way", - ]; - - /// Zvbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)] - pub const ZVBC_NAMES: &[&str] = &[ - "riscv64/zvbc", - "riscv64/zvbc-2way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - ]; - - // CRC64-XZ Kernel Function Arrays - // Note: riscv64 Zbc exposes up to 8-way; slot 4 is duplicated for index consistency. - - /// XZ Zbc kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)] - pub const XZ_ZBC: [Crc64Fn; 5] = [ - arch::crc64_xz_zbc_safe, - arch::crc64_xz_zbc_2way_safe, - arch::crc64_xz_zbc_4way_safe, - arch::crc64_xz_zbc_8way_safe, - arch::crc64_xz_zbc_8way_safe, // dup for index consistency - ]; - - /// XZ Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)] - pub const XZ_ZVBC: [Crc64Fn; 5] = [ - arch::crc64_xz_zvbc_safe, - arch::crc64_xz_zvbc_2way_safe, - arch::crc64_xz_zvbc_4way_safe, - arch::crc64_xz_zvbc_4way_safe, // dup for index consistency - arch::crc64_xz_zvbc_4way_safe, // dup for index consistency - ]; - - // CRC64-NVME Kernel Function Arrays - - /// NVME Zbc kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)] - pub const NVME_ZBC: [Crc64Fn; 5] = [ - arch::crc64_nvme_zbc_safe, - arch::crc64_nvme_zbc_2way_safe, - arch::crc64_nvme_zbc_4way_safe, - arch::crc64_nvme_zbc_8way_safe, - arch::crc64_nvme_zbc_8way_safe, // dup for index consistency - ]; - - /// NVME Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)] - pub const NVME_ZVBC: [Crc64Fn; 5] = [ - arch::crc64_nvme_zvbc_safe, - arch::crc64_nvme_zvbc_2way_safe, - arch::crc64_nvme_zvbc_4way_safe, - arch::crc64_nvme_zvbc_4way_safe, // dup for index consistency - arch::crc64_nvme_zvbc_4way_safe, // dup for index consistency - ]; -} diff --git a/src/checksum/crc64/mod.rs b/src/checksum/crc64/mod.rs index 4954d8cc..b50fb167 100644 --- a/src/checksum/crc64/mod.rs +++ b/src/checksum/crc64/mod.rs @@ -10,7 +10,6 @@ //! - aarch64: PMULL folding //! - Power: VPMSUMD folding //! - s390x: VGFM folding -//! - riscv64: ZVBC (RVV vector CLMUL) / Zbc folding //! - wasm32/wasm64: portable only (no CLMUL) pub(crate) mod config; @@ -33,11 +32,7 @@ mod power; #[cfg(target_arch = "s390x")] mod s390x; -#[cfg(target_arch = "riscv64")] -mod riscv64; - // Re-export config types for public API (Crc64Force only used internally on SIMD archs) -#[allow(unused_imports)] pub use config::{Crc64Config, Crc64Force}; #[cfg(any(test, feature = "std"))] @@ -47,10 +42,6 @@ use crate::checksum::common::tables::generate_crc64_tables_8; use crate::checksum::common::tables::{CRC64_NVME_POLY, CRC64_XZ_POLY, generate_crc64_tables_16}; #[cfg(feature = "diag")] use crate::checksum::diag::{Crc64Polynomial, Crc64SelectionDiag}; -// Re-export traits for test module (`use super::*`). -#[allow(unused_imports)] -pub(super) use crate::traits::{Checksum, ChecksumCombine}; - // Kernel Name Introspection /// Get the name of the CRC-64/XZ kernel that would be selected for a given buffer length. @@ -206,23 +197,21 @@ pub(crate) fn diag_crc64_nvme(len: usize) -> Crc64SelectionDiag { mod kernel_tables { use super::*; #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] - pub static XZ_TABLES_8: [[u64; 256]; 8] = generate_crc64_tables_8(CRC64_XZ_POLY); + pub(super) static XZ_TABLES_8: [[u64; 256]; 8] = generate_crc64_tables_8(CRC64_XZ_POLY); #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] - pub static NVME_TABLES_8: [[u64; 256]; 8] = generate_crc64_tables_8(CRC64_NVME_POLY); - pub static XZ_TABLES_16: [[u64; 256]; 16] = generate_crc64_tables_16(CRC64_XZ_POLY); - pub static NVME_TABLES_16: [[u64; 256]; 16] = generate_crc64_tables_16(CRC64_NVME_POLY); + pub(super) static NVME_TABLES_8: [[u64; 256]; 8] = generate_crc64_tables_8(CRC64_NVME_POLY); + pub(super) static XZ_TABLES_16: [[u64; 256]; 16] = generate_crc64_tables_16(CRC64_XZ_POLY); + pub(super) static NVME_TABLES_16: [[u64; 256]; 16] = generate_crc64_tables_16(CRC64_NVME_POLY); } /// CRC-64-XZ portable kernel wrapper. -#[cfg(any(test, feature = "std"))] -#[cfg_attr(all(test, not(feature = "std")), allow(dead_code))] +#[cfg(feature = "std")] fn crc64_xz_portable(crc: u64, data: &[u8]) -> u64 { portable::crc64_slice16_xz(crc, data) } /// CRC-64-NVME portable kernel wrapper. -#[cfg(any(test, feature = "std"))] -#[cfg_attr(all(test, not(feature = "std")), allow(dead_code))] +#[cfg(feature = "std")] fn crc64_nvme_portable(crc: u64, data: &[u8]) -> u64 { portable::crc64_slice16_nvme(crc, data) } @@ -849,6 +838,9 @@ impl crate::traits::ChecksumCombine for Crc64Nvme { #[cfg(feature = "alloc")] impl Crc64Nvme { + /// Buffer many short updates before dispatching to the active CRC-64/NVME kernel. + /// + /// For large contiguous buffers, use [`Crc64Nvme`] directly. #[must_use] pub fn buffered() -> BufferedCrc64Nvme { BufferedCrc64Nvme::new() @@ -964,9 +956,18 @@ mod tests { use alloc::vec::Vec; use super::*; + use crate::traits::{Checksum, ChecksumCombine}; const TEST_DATA: &[u8] = b"123456789"; + fn patterned_data(len: usize, multiplier: u8) -> Vec { + (0u8..=u8::MAX) + .cycle() + .take(len) + .map(|byte| byte.wrapping_mul(multiplier)) + .collect() + } + #[test] fn test_crc64_xz_checksum() { // Standard test vector for CRC-64-XZ (ECMA-182) @@ -1087,11 +1088,7 @@ mod tests { /// regardless of whether the portable slice-by-8 or SIMD path is selected. #[test] fn test_crc64_various_lengths() { - // Generate predictable test data - let mut data = [0u8; 512]; - for (i, byte) in data.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(17).wrapping_add(i as u8); - } + let data = patterned_data(512, 18); // Test lengths around key thresholds: // - 0-15: always portable (below 16B lane minimum) @@ -1136,7 +1133,7 @@ mod tests { // Generate test data larger than threshold let size = threshold + 128; - let data: Vec = (0..size).map(|i| (i as u8).wrapping_mul(31)).collect(); + let data = patterned_data(size, 31); let oneshot = Crc64::checksum(&data); @@ -1237,7 +1234,7 @@ mod tests { let len = 4096usize; // Execute both variants to ensure the selected tier doesn't trap. - let data: Vec = (0..len).map(|i| (i as u8).wrapping_mul(13)).collect(); + let data = patterned_data(len, 13); let ours_xz = Crc64::checksum(&data); let ours_nvme = Crc64Nvme::checksum(&data); @@ -1335,11 +1332,7 @@ mod tests { #[cfg(feature = "alloc")] #[test] fn test_buffered_crc64_xz_mixed_sizes() { - // Generate test data - let mut data = [0u8; 1024]; - for (i, byte) in data.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(13); - } + let data = patterned_data(1024, 13); let expected = Crc64::checksum(&data); let mut buffered = BufferedCrc64::new(); diff --git a/src/checksum/crc64/portable.rs b/src/checksum/crc64/portable.rs index 1d9bb895..d2ccccc9 100644 --- a/src/checksum/crc64/portable.rs +++ b/src/checksum/crc64/portable.rs @@ -9,30 +9,28 @@ use crate::checksum::common::portable; // Polynomial-specific wrappers /// CRC-64-XZ slice-by-8 computation. -#[cfg_attr(miri, allow(dead_code))] #[inline] #[cfg(all(test, any(target_arch = "x86_64", target_arch = "aarch64")))] -pub fn crc64_slice8_xz(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_slice8_xz(crc: u64, data: &[u8]) -> u64 { crc64_slice8(crc, data, &kernel_tables::XZ_TABLES_8) } /// CRC-64-NVME slice-by-8 computation. -#[cfg_attr(miri, allow(dead_code))] #[inline] #[cfg(all(test, any(target_arch = "x86_64", target_arch = "aarch64")))] -pub fn crc64_slice8_nvme(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_slice8_nvme(crc: u64, data: &[u8]) -> u64 { crc64_slice8(crc, data, &kernel_tables::NVME_TABLES_8) } /// CRC-64-XZ slice-by-16 computation. #[inline] -pub fn crc64_slice16_xz(crc: u64, data: &[u8]) -> u64 { +pub(in crate::checksum) fn crc64_slice16_xz(crc: u64, data: &[u8]) -> u64 { crc64_slice16(crc, data, &kernel_tables::XZ_TABLES_16) } /// CRC-64-NVME slice-by-16 computation. #[inline] -pub fn crc64_slice16_nvme(crc: u64, data: &[u8]) -> u64 { +pub(in crate::checksum) fn crc64_slice16_nvme(crc: u64, data: &[u8]) -> u64 { crc64_slice16(crc, data, &kernel_tables::NVME_TABLES_16) } @@ -42,7 +40,7 @@ pub fn crc64_slice16_nvme(crc: u64, data: &[u8]) -> u64 { /// /// Uses one 256-entry table rather than the slice-by-16 table set. #[inline(always)] -pub fn crc64_xz_bytewise(crc: u64, data: &[u8]) -> u64 { +pub(in crate::checksum) fn crc64_xz_bytewise(crc: u64, data: &[u8]) -> u64 { crc64_bytewise(crc, data, &kernel_tables::XZ_TABLES_16[0]) } @@ -50,13 +48,12 @@ pub fn crc64_xz_bytewise(crc: u64, data: &[u8]) -> u64 { /// /// Uses one 256-entry table rather than the slice-by-16 table set. #[inline(always)] -pub fn crc64_nvme_bytewise(crc: u64, data: &[u8]) -> u64 { +pub(in crate::checksum) fn crc64_nvme_bytewise(crc: u64, data: &[u8]) -> u64 { crc64_bytewise(crc, data, &kernel_tables::NVME_TABLES_16[0]) } /// Update CRC-64 state using a byte-at-a-time lookup table. #[inline(always)] -#[allow(clippy::indexing_slicing)] // index is 0..=255 by mask, table is [u64; 256] fn crc64_bytewise(mut crc: u64, data: &[u8], table: &[u64; 256]) -> u64 { for &b in data { let index = ((crc ^ (b as u64)) & 0xFF) as usize; @@ -76,7 +73,7 @@ fn crc64_bytewise(mut crc: u64, data: &[u8], table: &[u64; 256]) -> u64 { /// * `tables` - 8 lookup tables (256 entries each) #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] #[inline] -pub fn crc64_slice8(crc: u64, data: &[u8], tables: &[[u64; 256]; 8]) -> u64 { +pub(super) fn crc64_slice8(crc: u64, data: &[u8], tables: &[[u64; 256]; 8]) -> u64 { portable::slice8_64(crc, data, tables) } @@ -88,7 +85,7 @@ pub fn crc64_slice8(crc: u64, data: &[u8], tables: &[[u64; 256]; 8]) -> u64 { /// * `data` - Input data /// * `tables` - 16 lookup tables (256 entries each) #[inline] -pub fn crc64_slice16(crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { +pub(super) fn crc64_slice16(crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { portable::slice16_64(crc, data, tables) } diff --git a/src/checksum/crc64/power.rs b/src/checksum/crc64/power.rs index 7451295b..8f51ee3a 100644 --- a/src/checksum/crc64/power.rs +++ b/src/checksum/crc64/power.rs @@ -8,10 +8,7 @@ //! Uses `unsafe` for Power SIMD + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher // SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, @@ -66,6 +63,10 @@ impl Simd { /// On `powerpc64le` this is a no-op. On big-endian `powerpc64`, we byte-swap /// each 64-bit lane so the folding algorithm sees the same lane values as on /// little-endian platforms. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn to_le(self) -> Self { @@ -90,6 +91,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -113,6 +117,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -131,6 +138,10 @@ impl Simd { } /// Fold 16 bytes: `(coeff.low ⊗ self.low) ⊕ (coeff.high ⊗ self.high)`. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -145,6 +156,10 @@ impl Simd { } /// Fold 8 bytes: `self.high ⊕ (coeff ⊗ self.low)`. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -162,6 +177,10 @@ impl Simd { } /// Barrett reduction to finalize the CRC. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -226,6 +245,9 @@ const NVME_COMBINE_8WAY: [(u64, u64); 7] = [ // Folding helpers +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline(always)] unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { // SAFETY: POWER8 vector crypto intrinsics/asm are available via this function's #[target_feature] @@ -253,6 +275,9 @@ unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -275,6 +300,9 @@ unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -308,6 +336,9 @@ unsafe fn update_simd(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -363,7 +394,7 @@ unsafe fn update_simd_2way( let mut i = 2; while i < even { fold_block_128(&mut s0, &blocks[i], coeff_256); - fold_block_128(&mut s1, &blocks[i + 1], coeff_256); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_256); i = i.strict_add(2); } @@ -387,6 +418,9 @@ unsafe fn update_simd_2way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -412,7 +446,7 @@ unsafe fn update_simd_4way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 4) * 4; + let aligned = blocks.len() & !3_usize; let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); @@ -466,9 +500,9 @@ unsafe fn update_simd_4way( let mut i = 4; while i < aligned { fold_block_128(&mut s0, &blocks[i], coeff_512); - fold_block_128(&mut s1, &blocks[i + 1], coeff_512); - fold_block_128(&mut s2, &blocks[i + 2], coeff_512); - fold_block_128(&mut s3, &blocks[i + 3], coeff_512); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_512); i = i.strict_add(4); } @@ -509,6 +543,9 @@ unsafe fn update_simd_4way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -534,7 +571,7 @@ unsafe fn update_simd_8way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len() & !7_usize; let coeff_1024 = Simd::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); @@ -633,13 +670,13 @@ unsafe fn update_simd_8way( let mut i = 8; while i < aligned { fold_block_128(&mut s0, &blocks[i], coeff_1024); - fold_block_128(&mut s1, &blocks[i + 1], coeff_1024); - fold_block_128(&mut s2, &blocks[i + 2], coeff_1024); - fold_block_128(&mut s3, &blocks[i + 3], coeff_1024); - fold_block_128(&mut s4, &blocks[i + 4], coeff_1024); - fold_block_128(&mut s5, &blocks[i + 5], coeff_1024); - fold_block_128(&mut s6, &blocks[i + 6], coeff_1024); - fold_block_128(&mut s7, &blocks[i + 7], coeff_1024); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_1024); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_1024); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_1024); + fold_block_128(&mut s4, &blocks[i.strict_add(4)], coeff_1024); + fold_block_128(&mut s5, &blocks[i.strict_add(5)], coeff_1024); + fold_block_128(&mut s6, &blocks[i.strict_add(6)], coeff_1024); + fold_block_128(&mut s7, &blocks[i.strict_add(7)], coeff_1024); i = i.strict_add(8); } @@ -718,6 +755,9 @@ unsafe fn update_simd_8way( // Public kernels (XZ + NVME) +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -739,6 +779,9 @@ unsafe fn crc64_vpmsum(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstant } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -772,6 +815,9 @@ unsafe fn crc64_vpmsum_2way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -800,6 +846,9 @@ unsafe fn crc64_vpmsum_4way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -1041,49 +1090,49 @@ pub(crate) unsafe fn crc64_nvme_vpmsum_8way(crc: u64, data: &[u8]) -> u64 { // Safe wrappers #[inline] -pub fn crc64_xz_vpmsum_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpmsum_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_xz_vpmsum(crc, data) } } #[inline] -pub fn crc64_xz_vpmsum_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpmsum_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_xz_vpmsum_2way(crc, data) } } #[inline] -pub fn crc64_xz_vpmsum_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpmsum_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_xz_vpmsum_4way(crc, data) } } #[inline] -pub fn crc64_xz_vpmsum_8way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpmsum_8way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_xz_vpmsum_8way(crc, data) } } #[inline] -pub fn crc64_nvme_vpmsum_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpmsum_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_nvme_vpmsum(crc, data) } } #[inline] -pub fn crc64_nvme_vpmsum_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpmsum_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_nvme_vpmsum_2way(crc, data) } } #[inline] -pub fn crc64_nvme_vpmsum_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpmsum_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_nvme_vpmsum_4way(crc, data) } } #[inline] -pub fn crc64_nvme_vpmsum_8way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpmsum_8way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_nvme_vpmsum_8way(crc, data) } } diff --git a/src/checksum/crc64/riscv64.rs b/src/checksum/crc64/riscv64.rs deleted file mode 100644 index 011ad1d5..00000000 --- a/src/checksum/crc64/riscv64.rs +++ /dev/null @@ -1,1547 +0,0 @@ -//! riscv64 hardware-accelerated CRC-64 kernels (XZ + NVME). -//! -//! This is a scalar Zbc (`clmul`/`clmulh`) implementation of the Intel/TiKV -//! folding algorithm for CRC-64/XZ and CRC-64/NVME. -//! -//! # Safety -//! -//! Uses `unsafe` for RISC-V inline assembly. Callers must ensure the required -//! CPU features are available before executing the accelerated path (the -//! dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] - -use core::{ - arch::asm, - ops::{BitXor, BitXorAssign}, -}; - -use crate::checksum::common::{ - clmul::{Crc64ClmulConstants, fold16_coeff_for_bytes}, - tables::{CRC64_NVME_POLY, CRC64_XZ_POLY}, -}; - -type Block = [u64; 16]; // 128 bytes (8×16B lanes) - -#[derive(Copy, Clone, Debug)] -struct Simd { - hi: u64, - lo: u64, -} - -impl BitXor for Simd { - type Output = Self; - - #[inline] - fn bitxor(self, other: Self) -> Self { - Self { - hi: self.hi ^ other.hi, - lo: self.lo ^ other.lo, - } - } -} - -impl BitXorAssign for Simd { - #[inline] - fn bitxor_assign(&mut self, other: Self) { - self.hi ^= other.hi; - self.lo ^= other.lo; - } -} - -impl Simd { - #[inline] - const fn new(high: u64, low: u64) -> Self { - Self { hi: high, lo: low } - } - - #[inline] - const fn low_64(self) -> u64 { - self.lo - } - - #[inline] - const fn high_64(self) -> u64 { - self.hi - } - - #[inline] - #[target_feature(enable = "zbc")] - unsafe fn clmul_lo(a: u64, b: u64) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let out: u64; - asm!( - "clmul {out}, {a}, {b}", - out = lateout(reg) out, - a = in(reg) a, - b = in(reg) b, - options(nomem, nostack, pure) - ); - out - } - } - - #[inline] - #[target_feature(enable = "zbc")] - unsafe fn clmul_hi(a: u64, b: u64) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let out: u64; - asm!( - "clmulh {out}, {a}, {b}", - out = lateout(reg) out, - a = in(reg) a, - b = in(reg) b, - options(nomem, nostack, pure) - ); - out - } - } - - #[inline] - #[target_feature(enable = "zbc")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - Self { - hi: Self::clmul_hi(a, b), - lo: Self::clmul_lo(a, b), - } - } - } - - /// Fold 16 bytes: `(coeff.low ⊗ self.low) ⊕ (coeff.high ⊗ self.high)`. - #[inline] - #[target_feature(enable = "zbc")] - unsafe fn fold_16(self, coeff: (u64, u64)) -> Self { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - Self::mul64(self.low_64(), coeff_low) ^ Self::mul64(self.high_64(), coeff_high) - } - } - - /// Fold 8 bytes: `self.high ⊕ (coeff ⊗ self.low)`. - #[inline] - #[target_feature(enable = "zbc")] - unsafe fn fold_8(self, coeff: u64) -> Self { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let prod = Self::mul64(self.low_64(), coeff); - prod ^ Self::new(0, self.high_64()) - } - } - - /// Barrett reduction to finalize the CRC. - #[inline] - #[target_feature(enable = "zbc")] - unsafe fn barrett(self, poly: u64, mu: u64) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = Self::clmul_lo(self.low_64(), mu); - let l = Self::mul64(t1, poly); - (self ^ l).high_64() ^ t1 - } - } -} - -// Multi-stream coefficients (compile-time) - -// 2-way: update step shifts by 2×128B = 256B. -const XZ_FOLD_256B: (u64, u64) = fold16_coeff_for_bytes(CRC64_XZ_POLY, 256); -const NVME_FOLD_256B: (u64, u64) = fold16_coeff_for_bytes(CRC64_NVME_POLY, 256); - -// 4-way: update step shifts by 4×128B = 512B, combine shifts by 3/2/1 blocks. -const XZ_FOLD_512B: (u64, u64) = fold16_coeff_for_bytes(CRC64_XZ_POLY, 512); -const NVME_FOLD_512B: (u64, u64) = fold16_coeff_for_bytes(CRC64_NVME_POLY, 512); -const XZ_COMBINE_4WAY: [(u64, u64); 3] = [ - fold16_coeff_for_bytes(CRC64_XZ_POLY, 384), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 256), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 128), -]; -const NVME_COMBINE_4WAY: [(u64, u64); 3] = [ - fold16_coeff_for_bytes(CRC64_NVME_POLY, 384), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 256), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 128), -]; - -// 8-way: update step shifts by 8×128B = 1024B, combine shifts by 7/6/5/4/3/2/1 blocks. -const XZ_FOLD_1024B: (u64, u64) = fold16_coeff_for_bytes(CRC64_XZ_POLY, 1024); -const NVME_FOLD_1024B: (u64, u64) = fold16_coeff_for_bytes(CRC64_NVME_POLY, 1024); -const XZ_COMBINE_8WAY: [(u64, u64); 7] = [ - fold16_coeff_for_bytes(CRC64_XZ_POLY, 896), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 768), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 640), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 512), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 384), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 256), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 128), -]; -const NVME_COMBINE_8WAY: [(u64, u64); 7] = [ - fold16_coeff_for_bytes(CRC64_NVME_POLY, 896), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 768), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 640), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 512), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 384), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 256), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 128), -]; - -// Load helpers - -#[inline(always)] -fn load_block(block: &Block) -> [Simd; 8] { - let mut out = [Simd::new(0, 0); 8]; - - let mut i = 0; - while i < 8 { - let low = u64::from_le(block[i * 2]); - let high = u64::from_le(block[i * 2 + 1]); - out[i] = Simd::new(high, low); - i = i.strict_add(1); - } - - out -} - -// Folding helpers - -#[inline] -#[target_feature(enable = "zbc")] -unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut acc = x[7]; - acc ^= x[0].fold_16(consts.tail_fold_16b[0]); - acc ^= x[1].fold_16(consts.tail_fold_16b[1]); - acc ^= x[2].fold_16(consts.tail_fold_16b[2]); - acc ^= x[3].fold_16(consts.tail_fold_16b[3]); - acc ^= x[4].fold_16(consts.tail_fold_16b[4]); - acc ^= x[5].fold_16(consts.tail_fold_16b[5]); - acc ^= x[6].fold_16(consts.tail_fold_16b[6]); - - acc.fold_8(consts.fold_8b).barrett(consts.poly, consts.mu) - } -} - -#[inline] -#[target_feature(enable = "zbc")] -unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: (u64, u64)) { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - x[0] = chunk[0] ^ x[0].fold_16(coeff); - x[1] = chunk[1] ^ x[1].fold_16(coeff); - x[2] = chunk[2] ^ x[2].fold_16(coeff); - x[3] = chunk[3] ^ x[3].fold_16(coeff); - x[4] = chunk[4] ^ x[4].fold_16(coeff); - x[5] = chunk[5] ^ x[5].fold_16(coeff); - x[6] = chunk[6] ^ x[6].fold_16(coeff); - x[7] = chunk[7] ^ x[7].fold_16(coeff); - } -} - -#[inline] -#[target_feature(enable = "zbc")] -unsafe fn xor_fold_stream(combined: &mut [Simd; 8], stream: &[Simd; 8], coeff: (u64, u64)) { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut lane = 0usize; - while lane < 8 { - combined[lane] ^= stream[lane].fold_16(coeff); - lane = lane.strict_add(1); - } - } -} - -#[target_feature(enable = "zbc")] -unsafe fn update_simd(state: u64, first: &Block, rest: &[Block], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut x = load_block(first); - - // XOR the initial CRC into the first lane. - x[0] ^= Simd::new(0, state); - - let coeff = consts.fold_128b; - for block in rest { - let chunk = load_block(block); - fold_block_128(&mut x, &chunk, coeff); - } - - fold_tail(x, consts) - } -} - -#[target_feature(enable = "zbc")] -unsafe fn update_simd_2way(state: u64, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } - - let even = blocks.len() & !1usize; - - let coeff_256 = fold_256b; - let coeff_128 = consts.fold_128b; - - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - - // Inject CRC into stream 0. - s0[0] ^= Simd::new(0, state); - - let mut i = 2; - while i < even { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - fold_block_128(&mut s0, &b0, coeff_256); - fold_block_128(&mut s1, &b1, coeff_256); - i = i.strict_add(2); - } - - // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). - let mut combined = s1; - combined[0] ^= s0[0].fold_16(coeff_128); - combined[1] ^= s0[1].fold_16(coeff_128); - combined[2] ^= s0[2].fold_16(coeff_128); - combined[3] ^= s0[3].fold_16(coeff_128); - combined[4] ^= s0[4].fold_16(coeff_128); - combined[5] ^= s0[5].fold_16(coeff_128); - combined[6] ^= s0[6].fold_16(coeff_128); - combined[7] ^= s0[7].fold_16(coeff_128); - - // Handle any remaining block (odd tail) sequentially. - if even != blocks.len() { - let tail = load_block(&blocks[even]); - fold_block_128(&mut combined, &tail, coeff_128); - } - - fold_tail(combined, consts) - } -} - -#[target_feature(enable = "zbc")] -unsafe fn update_simd_4way( - state: u64, - blocks: &[Block], - fold_512b: (u64, u64), - combine: &[(u64, u64); 3], - consts: &Crc64ClmulConstants, -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } - - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512 = fold_512b; - let coeff_128 = consts.fold_128b; - - let c384 = combine[0]; - let c256 = combine[1]; - let c128 = combine[2]; - - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - let mut s2 = load_block(&blocks[2]); - let mut s3 = load_block(&blocks[3]); - - // Inject CRC into stream 0. - s0[0] ^= Simd::new(0, state); - - let mut i = 4; - while i < aligned { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - let b2 = load_block(&blocks[i.strict_add(2)]); - let b3 = load_block(&blocks[i.strict_add(3)]); - fold_block_128(&mut s0, &b0, coeff_512); - fold_block_128(&mut s1, &b1, coeff_512); - fold_block_128(&mut s2, &b2, coeff_512); - fold_block_128(&mut s3, &b3, coeff_512); - i = i.strict_add(4); - } - - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - let mut combined = s3; - combined[0] ^= s2[0].fold_16(c128); - combined[1] ^= s2[1].fold_16(c128); - combined[2] ^= s2[2].fold_16(c128); - combined[3] ^= s2[3].fold_16(c128); - combined[4] ^= s2[4].fold_16(c128); - combined[5] ^= s2[5].fold_16(c128); - combined[6] ^= s2[6].fold_16(c128); - combined[7] ^= s2[7].fold_16(c128); - - combined[0] ^= s1[0].fold_16(c256); - combined[1] ^= s1[1].fold_16(c256); - combined[2] ^= s1[2].fold_16(c256); - combined[3] ^= s1[3].fold_16(c256); - combined[4] ^= s1[4].fold_16(c256); - combined[5] ^= s1[5].fold_16(c256); - combined[6] ^= s1[6].fold_16(c256); - combined[7] ^= s1[7].fold_16(c256); - - combined[0] ^= s0[0].fold_16(c384); - combined[1] ^= s0[1].fold_16(c384); - combined[2] ^= s0[2].fold_16(c384); - combined[3] ^= s0[3].fold_16(c384); - combined[4] ^= s0[4].fold_16(c384); - combined[5] ^= s0[5].fold_16(c384); - combined[6] ^= s0[6].fold_16(c384); - combined[7] ^= s0[7].fold_16(c384); - - for block in &blocks[aligned..] { - let b = load_block(block); - fold_block_128(&mut combined, &b, coeff_128); - } - - fold_tail(combined, consts) - } -} - -#[target_feature(enable = "zbc")] -unsafe fn update_simd_8way( - state: u64, - blocks: &[Block], - fold_1024b: (u64, u64), - combine: &[(u64, u64); 7], - consts: &Crc64ClmulConstants, -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 8 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } - - let aligned = blocks.len().strict_div(8).strict_mul(8); - let coeff_1024 = fold_1024b; - let coeff_128 = consts.fold_128b; - - let mut streams = [ - load_block(&blocks[0]), - load_block(&blocks[1]), - load_block(&blocks[2]), - load_block(&blocks[3]), - load_block(&blocks[4]), - load_block(&blocks[5]), - load_block(&blocks[6]), - load_block(&blocks[7]), - ]; - - // Inject CRC into stream 0. - streams[0][0] ^= Simd::new(0, state); - - let mut i = 8usize; - while i < aligned { - let chunks = [ - load_block(&blocks[i]), - load_block(&blocks[i.strict_add(1)]), - load_block(&blocks[i.strict_add(2)]), - load_block(&blocks[i.strict_add(3)]), - load_block(&blocks[i.strict_add(4)]), - load_block(&blocks[i.strict_add(5)]), - load_block(&blocks[i.strict_add(6)]), - load_block(&blocks[i.strict_add(7)]), - ]; - - let mut stream = 0usize; - while stream < 8 { - fold_block_128(&mut streams[stream], &chunks[stream], coeff_1024); - stream = stream.strict_add(1); - } - - i = i.strict_add(8); - } - - // Merge: A^7·s0 ⊕ A^6·s1 ⊕ A^5·s2 ⊕ A^4·s3 ⊕ A^3·s4 ⊕ A^2·s5 ⊕ A·s6 ⊕ s7. - let mut combined = streams[7]; - let mut stream = 0usize; - while stream < 7 { - xor_fold_stream(&mut combined, &streams[stream], combine[stream]); - stream = stream.strict_add(1); - } - - for block in &blocks[aligned..] { - let chunk = load_block(block); - fold_block_128(&mut combined, &chunk, coeff_128); - } - - fold_tail(combined, consts) - } -} - -// ZVBC (vector carryless multiply) backend - -#[inline] -fn load_block_split(block: &Block) -> ([u64; 8], [u64; 8]) { - let mut hi = [0u64; 8]; - let mut lo = [0u64; 8]; - - let mut i = 0; - while i < 8 { - lo[i] = u64::from_le(block[i * 2]); - hi[i] = u64::from_le(block[i * 2 + 1]); - i = i.strict_add(1); - } - - (hi, lo) -} - -/// Carryless multiply of two `u64` values using ZVBC (returns 128-bit result as `{hi, lo}`). -/// -/// # Safety -/// -/// Requires RISC-V `v` + `zvbc`. -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let lo: u64; - let hi: u64; - asm!( - "vsetivli zero, 1, e64, m1, ta, ma", - "vmv.v.x v0, {a}", - "vclmul.vx v1, v0, {b}", - "vclmulh.vx v2, v0, {b}", - "vmv.x.s {lo}, v1", - "vmv.x.s {hi}, v2", - a = in(reg) a, - b = in(reg) b, - lo = lateout(reg) lo, - hi = lateout(reg) hi, - out("v0") _, - out("v1") _, - out("v2") _, - options(nostack) - ); - Simd::new(hi, lo) - } -} - -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - mul64_zvbc(x.low_64(), coeff_low) ^ mul64_zvbc(x.high_64(), coeff_high) - } -} - -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_8_zvbc(x: Simd, coeff: u64) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let prod = mul64_zvbc(x.low_64(), coeff); - Simd::new(prod.high_64(), prod.low_64() ^ x.high_64()) - } -} - -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn barrett_zvbc(x: Simd, poly: u64, mu: u64) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = mul64_zvbc(x.low_64(), mu).low_64(); - let l = mul64_zvbc(t1, poly); - (x ^ l).high_64() ^ t1 - } -} - -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut acc = Simd::new(hi[7], lo[7]); - acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), consts.tail_fold_16b[0]); - acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), consts.tail_fold_16b[1]); - acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), consts.tail_fold_16b[2]); - acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), consts.tail_fold_16b[3]); - acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), consts.tail_fold_16b[4]); - acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), consts.tail_fold_16b[5]); - acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), consts.tail_fold_16b[6]); - barrett_zvbc(fold_8_zvbc(acc, consts.fold_8b), consts.poly, consts.mu) - } -} - -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_block_128_zvbc( - x_hi: &mut [u64; 8], - x_lo: &mut [u64; 8], - chunk_hi: &[u64; 8], - chunk_lo: &[u64; 8], - coeff_low: u64, - coeff_high: u64, -) { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut offset = 0usize; - while offset < 8 { - let remaining = 8 - offset; - let vl: usize; - asm!( - "vsetvli {vl}, {avl}, e64, m1, ta, ma", - "vle64.v v0, ({xlo})", - "vle64.v v1, ({xhi})", - "vclmul.vx v2, v0, {clo}", - "vclmulh.vx v3, v0, {clo}", - "vclmul.vx v4, v1, {chi}", - "vclmulh.vx v5, v1, {chi}", - "vxor.vv v2, v2, v4", - "vxor.vv v3, v3, v5", - "vle64.v v4, ({dlo})", - "vle64.v v5, ({dhi})", - "vxor.vv v2, v2, v4", - "vxor.vv v3, v3, v5", - "vse64.v v2, ({xlo})", - "vse64.v v3, ({xhi})", - vl = lateout(reg) vl, - avl = in(reg) remaining, - xlo = in(reg) x_lo.as_mut_ptr().add(offset), - xhi = in(reg) x_hi.as_mut_ptr().add(offset), - dlo = in(reg) chunk_lo.as_ptr().add(offset), - dhi = in(reg) chunk_hi.as_ptr().add(offset), - clo = in(reg) coeff_low, - chi = in(reg) coeff_high, - out("v0") _, - out("v1") _, - out("v2") _, - out("v3") _, - out("v4") _, - out("v5") _, - options(nostack) - ); - offset = offset.strict_add(vl); - } - } -} - -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc(state: u64, first: &Block, rest: &[Block], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (mut x_hi, mut x_lo) = load_block_split(first); - - // XOR the initial CRC into the first lane. - x_lo[0] ^= state; - - let coeff_low = consts.fold_128b.1; - let coeff_high = consts.fold_128b.0; - - for block in rest { - let (chunk_hi, chunk_lo) = load_block_split(block); - fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); - } - - fold_tail_zvbc(x_hi, x_lo, consts) - } -} - -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_2way( - state: u64, - blocks: &[Block], - fold_256b: (u64, u64), - consts: &Crc64ClmulConstants, -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, consts); - } - - let even = blocks.len() & !1usize; - - let coeff_256_low = fold_256b.1; - let coeff_256_high = fold_256b.0; - let coeff_128_low = consts.fold_128b.1; - let coeff_128_high = consts.fold_128b.0; - - let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); - - // Inject CRC into stream 0. - s0_lo[0] ^= state; - - let mut i = 2; - while i < even { - let (b0_hi, b0_lo) = load_block_split(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); - i = i.strict_add(2); - } - - // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, coeff_128_low, coeff_128_high); - - // Handle any remaining block (odd tail) sequentially. - if even != blocks.len() { - let (tail_hi, tail_lo) = load_block_split(&blocks[even]); - fold_block_128_zvbc( - &mut s0_hi, - &mut s0_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } - - fold_tail_zvbc(s0_hi, s0_lo, consts) - } -} - -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_4way( - state: u64, - blocks: &[Block], - fold_512b: (u64, u64), - combine: &[(u64, u64); 3], - consts: &Crc64ClmulConstants, -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, consts); - } - - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512_low = fold_512b.1; - let coeff_512_high = fold_512b.0; - let coeff_128_low = consts.fold_128b.1; - let coeff_128_high = consts.fold_128b.0; - - let c384_low = combine[0].1; - let c384_high = combine[0].0; - let c256_low = combine[1].1; - let c256_high = combine[1].0; - let c128_low = combine[2].1; - let c128_high = combine[2].0; - - let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); - let (mut s2_hi, mut s2_lo) = load_block_split(&blocks[2]); - let (mut s3_hi, mut s3_lo) = load_block_split(&blocks[3]); - - // Inject CRC into stream 0. - s0_lo[0] ^= state; - - let mut i = 4; - while i < aligned { - let (b0_hi, b0_lo) = load_block_split(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); - let (b2_hi, b2_lo) = load_block_split(&blocks[i.strict_add(2)]); - let (b3_hi, b3_lo) = load_block_split(&blocks[i.strict_add(3)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); - i = i.strict_add(4); - } - - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - // - // `fold_block_128_zvbc(x, chunk, c)` computes: `x = chunk ⊕ fold(x, c)`. - // We want: `combined = combined ⊕ fold(stream, c)`, so we fold each stream - // into the current combined value by using `chunk = combined` and storing the - // result back into that stream (which is no longer needed after this point). - let mut combined_hi = s3_hi; - let mut combined_lo = s3_lo; - - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &combined_hi, &combined_lo, c128_low, c128_high); - combined_hi = s2_hi; - combined_lo = s2_lo; - - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &combined_hi, &combined_lo, c256_low, c256_high); - combined_hi = s1_hi; - combined_lo = s1_lo; - - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &combined_hi, &combined_lo, c384_low, c384_high); - combined_hi = s0_hi; - combined_lo = s0_lo; - - for block in &blocks[aligned..] { - let (tail_hi, tail_lo) = load_block_split(block); - fold_block_128_zvbc( - &mut combined_hi, - &mut combined_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } - - fold_tail_zvbc(combined_hi, combined_lo, consts) - } -} - -// Public kernels (XZ + NVME) - -#[target_feature(enable = "zbc")] -unsafe fn crc64_zbc(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 16]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - // SAFETY: `blocks_u64` length is a multiple of 16, so casting to `[u64; 16]` is safe. - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - if let Some((first, rest)) = blocks.split_first() { - state = update_simd(state, first, rest, consts); - } - } - - if !tail_u64.is_empty() { - // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc64_zvbc(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 16]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - // SAFETY: `blocks_u64` length is a multiple of 16, so casting to `[u64; 16]` is safe. - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - if let Some((first, rest)) = blocks.split_first() { - state = update_simd_zvbc(state, first, rest, consts); - } - } - - if !tail_u64.is_empty() { - // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -#[target_feature(enable = "zbc")] -unsafe fn crc64_zbc_2way( - mut state: u64, - bytes: &[u8], - fold_256b: (u64, u64), - consts: &Crc64ClmulConstants, - tables: &[[u64; 256]; 16], -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - // SAFETY: `blocks_u64` length is a multiple of 16, so casting to `[u64; 16]` is safe. - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - state = update_simd_2way(state, blocks, fold_256b, consts); - } - - if !tail_u64.is_empty() { - // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc64_zvbc_2way( - mut state: u64, - bytes: &[u8], - fold_256b: (u64, u64), - consts: &Crc64ClmulConstants, - tables: &[[u64; 256]; 16], -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - // SAFETY: `blocks_u64` length is a multiple of 16, so casting to `[u64; 16]` is safe. - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - state = update_simd_zvbc_2way(state, blocks, fold_256b, consts); - } - - if !tail_u64.is_empty() { - // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -#[target_feature(enable = "zbc")] -unsafe fn crc64_zbc_4way( - mut state: u64, - bytes: &[u8], - fold_512b: (u64, u64), - combine: &[(u64, u64); 3], - consts: &Crc64ClmulConstants, - tables: &[[u64; 256]; 16], -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - // SAFETY: `blocks_u64` length is a multiple of 16, so casting to `[u64; 16]` is safe. - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - state = update_simd_4way(state, blocks, fold_512b, combine, consts); - } - - if !tail_u64.is_empty() { - // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -#[target_feature(enable = "zbc")] -unsafe fn crc64_zbc_8way( - mut state: u64, - bytes: &[u8], - fold_1024b: (u64, u64), - combine: &[(u64, u64); 7], - consts: &Crc64ClmulConstants, - tables: &[[u64; 256]; 16], -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - state = update_simd_8way(state, blocks, fold_1024b, combine, consts); - } - - if !tail_u64.is_empty() { - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc64_zvbc_4way( - mut state: u64, - bytes: &[u8], - fold_512b: (u64, u64), - combine: &[(u64, u64); 3], - consts: &Crc64ClmulConstants, - tables: &[[u64; 256]; 16], -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - // SAFETY: `blocks_u64` length is a multiple of 16, so casting to `[u64; 16]` is safe. - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - state = update_simd_zvbc_4way(state, blocks, fold_512b, combine, consts); - } - - if !tail_u64.is_empty() { - // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -/// CRC-64-XZ using scalar Zbc carryless multiply folding. -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_xz_zbc(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc( - crc, - data, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-XZ using RVV Zvbc (vector carryless multiply) folding. -/// -/// # Safety -/// -/// Requires the RISC-V vector extension and Zvbc. Caller must verify via -/// `crate::platform::caps().has(riscv::ZVBC)`. -#[target_feature(enable = "v", enable = "zvbc")] -pub(crate) unsafe fn crc64_xz_zvbc(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zvbc( - crc, - data, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-XZ using scalar Zbc carryless multiply folding (2-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_xz_zbc_2way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc_2way( - crc, - data, - XZ_FOLD_256B, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-XZ using RVV Zvbc (vector carryless multiply) folding (2-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V vector extension and Zvbc. Caller must verify via -/// `crate::platform::caps().has(riscv::ZVBC)`. -#[target_feature(enable = "v", enable = "zvbc")] -pub(crate) unsafe fn crc64_xz_zvbc_2way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zvbc_2way( - crc, - data, - XZ_FOLD_256B, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-XZ using scalar Zbc carryless multiply folding (4-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_xz_zbc_4way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc_4way( - crc, - data, - XZ_FOLD_512B, - &XZ_COMBINE_4WAY, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-XZ using scalar Zbc carryless multiply folding (8-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_xz_zbc_8way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc_8way( - crc, - data, - XZ_FOLD_1024B, - &XZ_COMBINE_8WAY, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-XZ using RVV Zvbc (vector carryless multiply) folding (4-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V vector extension and Zvbc. Caller must verify via -/// `crate::platform::caps().has(riscv::ZVBC)`. -#[target_feature(enable = "v", enable = "zvbc")] -pub(crate) unsafe fn crc64_xz_zvbc_4way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zvbc_4way( - crc, - data, - XZ_FOLD_512B, - &XZ_COMBINE_4WAY, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-NVME using scalar Zbc carryless multiply folding. -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_nvme_zbc(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc( - crc, - data, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -/// CRC-64-NVME using RVV Zvbc (vector carryless multiply) folding. -/// -/// # Safety -/// -/// Requires the RISC-V vector extension and Zvbc. Caller must verify via -/// `crate::platform::caps().has(riscv::ZVBC)`. -#[target_feature(enable = "v", enable = "zvbc")] -pub(crate) unsafe fn crc64_nvme_zvbc(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zvbc( - crc, - data, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -/// CRC-64-NVME using scalar Zbc carryless multiply folding (2-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_nvme_zbc_2way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc_2way( - crc, - data, - NVME_FOLD_256B, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -/// CRC-64-NVME using RVV Zvbc (vector carryless multiply) folding (2-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V vector extension and Zvbc. Caller must verify via -/// `crate::platform::caps().has(riscv::ZVBC)`. -#[target_feature(enable = "v", enable = "zvbc")] -pub(crate) unsafe fn crc64_nvme_zvbc_2way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zvbc_2way( - crc, - data, - NVME_FOLD_256B, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -/// CRC-64-NVME using scalar Zbc carryless multiply folding (4-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_nvme_zbc_4way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc_4way( - crc, - data, - NVME_FOLD_512B, - &NVME_COMBINE_4WAY, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -/// CRC-64-NVME using scalar Zbc carryless multiply folding (8-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_nvme_zbc_8way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc_8way( - crc, - data, - NVME_FOLD_1024B, - &NVME_COMBINE_8WAY, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -/// CRC-64-NVME using RVV Zvbc (vector carryless multiply) folding (4-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V vector extension and Zvbc. Caller must verify via -/// `crate::platform::caps().has(riscv::ZVBC)`. -#[target_feature(enable = "v", enable = "zvbc")] -pub(crate) unsafe fn crc64_nvme_zvbc_4way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zvbc_4way( - crc, - data, - NVME_FOLD_512B, - &NVME_COMBINE_4WAY, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -// Safe wrappers - -#[inline] -pub fn crc64_xz_zbc_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zbc(crc, data) } -} - -#[inline] -pub fn crc64_xz_zvbc_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zvbc(crc, data) } -} - -#[inline] -pub fn crc64_xz_zbc_2way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zbc_2way(crc, data) } -} - -#[inline] -pub fn crc64_xz_zvbc_2way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zvbc_2way(crc, data) } -} - -#[inline] -pub fn crc64_xz_zbc_4way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zbc_4way(crc, data) } -} - -#[inline] -pub fn crc64_xz_zbc_8way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zbc_8way(crc, data) } -} - -#[inline] -pub fn crc64_xz_zvbc_4way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zvbc_4way(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zbc_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zbc(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zvbc_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zvbc(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zbc_2way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zbc_2way(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zvbc_2way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zvbc_2way(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zbc_4way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zbc_4way(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zbc_8way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zbc_8way(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zvbc_4way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zvbc_4way(crc, data) } -} - -#[cfg(test)] -mod tests { - extern crate std; - - use alloc::vec::Vec; - - use super::*; - - const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; - const OFFSETS: &[usize] = &[0, 1, 7, 15]; - const STATES: &[u64] = &[0, 0x0123_4567_89ab_cdef, 0xa5a5_5a5a_dead_beef, u64::MAX]; - - fn assert_kernel(name: &str, kernel: fn(u64, &[u8]) -> u64, portable: fn(u64, &[u8]) -> u64) { - let input: Vec = (0..4111) - .map(|i| (i as u8).wrapping_mul(41).wrapping_add((i >> 8) as u8)) - .collect(); - for &state in STATES { - for &offset in OFFSETS { - for &len in LENS { - let slice = &input[offset..offset + len]; - assert_eq!( - kernel(state, slice), - portable(state, slice), - "{name} state={state:#018x} offset={offset} len={len}" - ); - } - } - } - } - - #[test] - fn zbc_kernels_match_portable() { - if !crate::platform::caps().has(crate::platform::caps::riscv::ZBC) { - return; - } - - for (name, kernel) in [ - ("xz/zbc", crc64_xz_zbc_safe as fn(u64, &[u8]) -> u64), - ("xz/zbc-2way", crc64_xz_zbc_2way_safe), - ("xz/zbc-4way", crc64_xz_zbc_4way_safe), - ("xz/zbc-8way", crc64_xz_zbc_8way_safe), - ] { - assert_kernel(name, kernel, super::super::portable::crc64_slice16_xz); - } - - for (name, kernel) in [ - ("nvme/zbc", crc64_nvme_zbc_safe as fn(u64, &[u8]) -> u64), - ("nvme/zbc-2way", crc64_nvme_zbc_2way_safe), - ("nvme/zbc-4way", crc64_nvme_zbc_4way_safe), - ("nvme/zbc-8way", crc64_nvme_zbc_8way_safe), - ] { - assert_kernel(name, kernel, super::super::portable::crc64_slice16_nvme); - } - } - - #[test] - fn zvbc_kernels_match_portable() { - use crate::platform::caps::riscv; - - if !crate::platform::caps().has(riscv::V.union(riscv::ZVBC)) { - return; - } - - for (name, kernel) in [ - ("xz/zvbc", crc64_xz_zvbc_safe as fn(u64, &[u8]) -> u64), - ("xz/zvbc-2way", crc64_xz_zvbc_2way_safe), - ("xz/zvbc-4way", crc64_xz_zvbc_4way_safe), - ] { - assert_kernel(name, kernel, super::super::portable::crc64_slice16_xz); - } - - for (name, kernel) in [ - ("nvme/zvbc", crc64_nvme_zvbc_safe as fn(u64, &[u8]) -> u64), - ("nvme/zvbc-2way", crc64_nvme_zvbc_2way_safe), - ("nvme/zvbc-4way", crc64_nvme_zvbc_4way_safe), - ] { - assert_kernel(name, kernel, super::super::portable::crc64_slice16_nvme); - } - } -} diff --git a/src/checksum/crc64/s390x.rs b/src/checksum/crc64/s390x.rs index c806e28d..29f517c4 100644 --- a/src/checksum/crc64/s390x.rs +++ b/src/checksum/crc64/s390x.rs @@ -8,15 +8,9 @@ //! Uses `unsafe` for s390x vector + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] -// This module is intrinsics-heavy; unsafe blocks are per-function with SAFETY justifications. use core::{ arch::asm, - mem::MaybeUninit, ops::{BitXor, BitXorAssign}, simd::i64x2, }; @@ -66,8 +60,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn vgfm(a: i64x2, b: i64x2) -> i64x2 { + fn vgfm(a: i64x2, b: i64x2) -> i64x2 { // SAFETY: Caller guarantees the s390x vector facility is available // (verified by dispatch). The VGFM instruction operates on pure register // values with no memory access. @@ -85,49 +82,47 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Caller guarantees the s390x vector facility is available. - // vgfm operates on pure register values. - unsafe { - let va = Self::new(0, a); - let vb = Self::new(0, b); - Self(Self::vgfm(va.0, vb.0)) - } + fn mul64(a: u64, b: u64) -> Self { + let va = Self::new(0, a); + let vb = Self::new(0, b); + Self(Self::vgfm(va.0, vb.0)) } /// Fold 16 bytes: `(coeff.low ⊗ self.low) ⊕ (coeff.high ⊗ self.high)`. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_16(self, coeff: Self) -> Self { - // SAFETY: Caller guarantees the s390x vector facility is available. - // vgfm operates on pure register values. - unsafe { Self(Self::vgfm(self.0, coeff.0)) } + fn fold_16(self, coeff: Self) -> Self { + Self(Self::vgfm(self.0, coeff.0)) } /// Fold 8 bytes: `self.high ⊕ (coeff ⊗ self.low)`. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_8(self, coeff: u64) -> Self { - // SAFETY: Caller guarantees the s390x vector facility is available. - // mul64 and XOR are pure register computations. - unsafe { - let prod = Self::mul64(self.low_64(), coeff); - prod ^ Self::new(0, self.high_64()) - } + fn fold_8(self, coeff: u64) -> Self { + let prod = Self::mul64(self.low_64(), coeff); + prod ^ Self::new(0, self.high_64()) } /// Barrett reduction to finalize the CRC. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn barrett(self, poly: u64, mu: u64) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available. - // mul64 and XOR are pure register computations. - unsafe { - let t1 = Self::mul64(self.low_64(), mu).low_64(); - let l = Self::mul64(t1, poly); - (self ^ l).high_64() ^ t1 - } + fn barrett(self, poly: u64, mu: u64) -> u64 { + let t1 = Self::mul64(self.low_64(), mu).low_64(); + let l = Self::mul64(t1, poly); + (self ^ l).high_64() ^ t1 } } @@ -155,153 +150,136 @@ const NVME_COMBINE_4WAY: [(u64, u64); 3] = [ #[inline(always)] fn load_block(block: &Block) -> [Simd; 8] { - let mut out = MaybeUninit::<[Simd; 8]>::uninit(); - let base = out.as_mut_ptr().cast::(); - - let mut i = 0; - while i < 8 { - let low = u64::from_le(block[i * 2]); - let high = u64::from_le(block[i * 2 + 1]); - // SAFETY: `base` points to a `[Simd; 8]` buffer and `i` is in-bounds. - unsafe { - base.add(i).write(Simd::new(high, low)); - } - i = i.strict_add(1); + let mut out = [Simd::new(0, 0); 8]; + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new(u64::from_le(high), u64::from_le(low)); } - - // SAFETY: all 8 elements are initialized above. - unsafe { out.assume_init() } + out } // Folding helpers -#[inline(always)] -unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold_16 / fold_8 / barrett calls are pure VGFM register computations. - // Array indexing is over fixed-size [Simd; 8] with constant indices 0..7. - unsafe { - // Tail reduction (8×16B → 1×16B), unrolled for throughput. - let c0 = Simd::new(consts.tail_fold_16b[0].0, consts.tail_fold_16b[0].1); - let c1 = Simd::new(consts.tail_fold_16b[1].0, consts.tail_fold_16b[1].1); - let c2 = Simd::new(consts.tail_fold_16b[2].0, consts.tail_fold_16b[2].1); - let c3 = Simd::new(consts.tail_fold_16b[3].0, consts.tail_fold_16b[3].1); - let c4 = Simd::new(consts.tail_fold_16b[4].0, consts.tail_fold_16b[4].1); - let c5 = Simd::new(consts.tail_fold_16b[5].0, consts.tail_fold_16b[5].1); - let c6 = Simd::new(consts.tail_fold_16b[6].0, consts.tail_fold_16b[6].1); - - let mut acc = x[7]; - acc ^= x[0].fold_16(c0); - acc ^= x[1].fold_16(c1); - acc ^= x[2].fold_16(c2); - acc ^= x[3].fold_16(c3); - acc ^= x[4].fold_16(c4); - acc ^= x[5].fold_16(c5); - acc ^= x[6].fold_16(c6); - - acc.fold_8(consts.fold_8b).barrett(consts.poly, consts.mu) - } +#[inline] +/// # Safety +/// +/// Requires the s390x vector facility. +#[target_feature(enable = "vector")] +fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { + let c0 = Simd::new(consts.tail_fold_16b[0].0, consts.tail_fold_16b[0].1); + let c1 = Simd::new(consts.tail_fold_16b[1].0, consts.tail_fold_16b[1].1); + let c2 = Simd::new(consts.tail_fold_16b[2].0, consts.tail_fold_16b[2].1); + let c3 = Simd::new(consts.tail_fold_16b[3].0, consts.tail_fold_16b[3].1); + let c4 = Simd::new(consts.tail_fold_16b[4].0, consts.tail_fold_16b[4].1); + let c5 = Simd::new(consts.tail_fold_16b[5].0, consts.tail_fold_16b[5].1); + let c6 = Simd::new(consts.tail_fold_16b[6].0, consts.tail_fold_16b[6].1); + + let mut acc = x[7]; + acc ^= x[0].fold_16(c0); + acc ^= x[1].fold_16(c1); + acc ^= x[2].fold_16(c2); + acc ^= x[3].fold_16(c3); + acc ^= x[4].fold_16(c4); + acc ^= x[5].fold_16(c5); + acc ^= x[6].fold_16(c6); + + acc.fold_8(consts.fold_8b).barrett(consts.poly, consts.mu) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold_16 calls are pure VGFM register computations. - // Array indexing is over fixed-size [Simd; 8] with constant indices 0..7. - unsafe { - x[0] = chunk[0] ^ x[0].fold_16(coeff); - x[1] = chunk[1] ^ x[1].fold_16(coeff); - x[2] = chunk[2] ^ x[2].fold_16(coeff); - x[3] = chunk[3] ^ x[3].fold_16(coeff); - x[4] = chunk[4] ^ x[4].fold_16(coeff); - x[5] = chunk[5] ^ x[5].fold_16(coeff); - x[6] = chunk[6] ^ x[6].fold_16(coeff); - x[7] = chunk[7] ^ x[7].fold_16(coeff); - } +fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { + x[0] = chunk[0] ^ x[0].fold_16(coeff); + x[1] = chunk[1] ^ x[1].fold_16(coeff); + x[2] = chunk[2] ^ x[2].fold_16(coeff); + x[3] = chunk[3] ^ x[3].fold_16(coeff); + x[4] = chunk[4] ^ x[4].fold_16(coeff); + x[5] = chunk[5] ^ x[5].fold_16(coeff); + x[6] = chunk[6] ^ x[6].fold_16(coeff); + x[7] = chunk[7] ^ x[7].fold_16(coeff); } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd(state: u64, first: &Block, rest: &[Block], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold operations are pure VGFM register computations. - // Block iteration is bounded by the slice length; no out-of-bounds access. - unsafe { - let mut x = load_block(first); +fn update_simd(state: u64, first: &Block, rest: &[Block], consts: &Crc64ClmulConstants) -> u64 { + let mut x = load_block(first); - // XOR the initial CRC into the first lane. - x[0] ^= Simd::new(0, state); + // XOR the initial CRC into the first lane. + x[0] ^= Simd::new(0, state); - // 128-byte folding. - let coeff = Simd::new(consts.fold_128b.0, consts.fold_128b.1); - for block in rest { - let chunk = load_block(block); - fold_block_128(&mut x, &chunk, coeff); - } - - fold_tail(x, consts) + // 128-byte folding. + let coeff = Simd::new(consts.fold_128b.0, consts.fold_128b.1); + for block in rest { + let chunk = load_block(block); + fold_block_128(&mut x, &chunk, coeff); } + + fold_tail(x, consts) } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_2way(state: u64, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc64ClmulConstants) -> u64 { +fn update_simd_2way(state: u64, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc64ClmulConstants) -> u64 { debug_assert!(!blocks.is_empty()); - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold operations are pure VGFM register computations. - // Loop index `i` advances by 2 and is bounded by `even` (≤ blocks.len()), - // so `blocks[i]` and `blocks[i + 1]` are always in-bounds. - unsafe { - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd(state, first, rest, consts); + } - let even = blocks.len() & !1usize; + let even = blocks.len() & !1usize; - let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); - let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); + let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); + let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); - // Inject CRC into stream 0. - s0[0] ^= Simd::new(0, state); + // Inject CRC into stream 0. + s0[0] ^= Simd::new(0, state); - let mut i = 2; - while i < even { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - fold_block_128(&mut s0, &b0, coeff_256); - fold_block_128(&mut s1, &b1, coeff_256); - i = i.strict_add(2); - } + let mut i = 2; + while i < even { + let b0 = load_block(&blocks[i]); + let b1 = load_block(&blocks[i.strict_add(1)]); + fold_block_128(&mut s0, &b0, coeff_256); + fold_block_128(&mut s1, &b1, coeff_256); + i = i.strict_add(2); + } - // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). - let mut combined = s1; - combined[0] ^= s0[0].fold_16(coeff_128); - combined[1] ^= s0[1].fold_16(coeff_128); - combined[2] ^= s0[2].fold_16(coeff_128); - combined[3] ^= s0[3].fold_16(coeff_128); - combined[4] ^= s0[4].fold_16(coeff_128); - combined[5] ^= s0[5].fold_16(coeff_128); - combined[6] ^= s0[6].fold_16(coeff_128); - combined[7] ^= s0[7].fold_16(coeff_128); - - // Handle any remaining block (odd tail) sequentially. - if even != blocks.len() { - let tail = load_block(&blocks[even]); - fold_block_128(&mut combined, &tail, coeff_128); - } + // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). + let mut combined = s1; + combined[0] ^= s0[0].fold_16(coeff_128); + combined[1] ^= s0[1].fold_16(coeff_128); + combined[2] ^= s0[2].fold_16(coeff_128); + combined[3] ^= s0[3].fold_16(coeff_128); + combined[4] ^= s0[4].fold_16(coeff_128); + combined[5] ^= s0[5].fold_16(coeff_128); + combined[6] ^= s0[6].fold_16(coeff_128); + combined[7] ^= s0[7].fold_16(coeff_128); + + // Handle any remaining block (odd tail) sequentially. + if even != blocks.len() { + let tail = load_block(&blocks[even]); + fold_block_128(&mut combined, &tail, coeff_128); + } - fold_tail(combined, consts) - } // unsafe + fold_tail(combined, consts) } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_4way( +fn update_simd_4way( state: u64, blocks: &[Block], fold_512b: (u64, u64), @@ -310,92 +288,88 @@ unsafe fn update_simd_4way( ) -> u64 { debug_assert!(!blocks.is_empty()); - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold operations are pure VGFM register computations. - // Loop index `i` advances by 4 and is bounded by `aligned` (≤ blocks.len()), - // so `blocks[i..i+3]` are always in-bounds. Tail processing uses iterator. - unsafe { - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd(state, first, rest, consts); + } - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); - let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); - let c384 = Simd::new(combine[0].0, combine[0].1); - let c256 = Simd::new(combine[1].0, combine[1].1); - let c128 = Simd::new(combine[2].0, combine[2].1); - - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - let mut s2 = load_block(&blocks[2]); - let mut s3 = load_block(&blocks[3]); - - // Inject CRC into stream 0. - s0[0] ^= Simd::new(0, state); - - let mut i = 4; - while i < aligned { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - let b2 = load_block(&blocks[i.strict_add(2)]); - let b3 = load_block(&blocks[i.strict_add(3)]); - fold_block_128(&mut s0, &b0, coeff_512); - fold_block_128(&mut s1, &b1, coeff_512); - fold_block_128(&mut s2, &b2, coeff_512); - fold_block_128(&mut s3, &b3, coeff_512); - i = i.strict_add(4); - } + let aligned = blocks.len().strict_div(4).strict_mul(4); + + let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); + let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); + let c384 = Simd::new(combine[0].0, combine[0].1); + let c256 = Simd::new(combine[1].0, combine[1].1); + let c128 = Simd::new(combine[2].0, combine[2].1); + + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + let mut s2 = load_block(&blocks[2]); + let mut s3 = load_block(&blocks[3]); + + // Inject CRC into stream 0. + s0[0] ^= Simd::new(0, state); + + let mut i = 4; + while i < aligned { + let b0 = load_block(&blocks[i]); + let b1 = load_block(&blocks[i.strict_add(1)]); + let b2 = load_block(&blocks[i.strict_add(2)]); + let b3 = load_block(&blocks[i.strict_add(3)]); + fold_block_128(&mut s0, &b0, coeff_512); + fold_block_128(&mut s1, &b1, coeff_512); + fold_block_128(&mut s2, &b2, coeff_512); + fold_block_128(&mut s3, &b3, coeff_512); + i = i.strict_add(4); + } - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - let mut combined = s3; - combined[0] ^= s2[0].fold_16(c128); - combined[1] ^= s2[1].fold_16(c128); - combined[2] ^= s2[2].fold_16(c128); - combined[3] ^= s2[3].fold_16(c128); - combined[4] ^= s2[4].fold_16(c128); - combined[5] ^= s2[5].fold_16(c128); - combined[6] ^= s2[6].fold_16(c128); - combined[7] ^= s2[7].fold_16(c128); - - combined[0] ^= s1[0].fold_16(c256); - combined[1] ^= s1[1].fold_16(c256); - combined[2] ^= s1[2].fold_16(c256); - combined[3] ^= s1[3].fold_16(c256); - combined[4] ^= s1[4].fold_16(c256); - combined[5] ^= s1[5].fold_16(c256); - combined[6] ^= s1[6].fold_16(c256); - combined[7] ^= s1[7].fold_16(c256); - - combined[0] ^= s0[0].fold_16(c384); - combined[1] ^= s0[1].fold_16(c384); - combined[2] ^= s0[2].fold_16(c384); - combined[3] ^= s0[3].fold_16(c384); - combined[4] ^= s0[4].fold_16(c384); - combined[5] ^= s0[5].fold_16(c384); - combined[6] ^= s0[6].fold_16(c384); - combined[7] ^= s0[7].fold_16(c384); - - for block in &blocks[aligned..] { - let b = load_block(block); - fold_block_128(&mut combined, &b, coeff_128); - } + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + let mut combined = s3; + combined[0] ^= s2[0].fold_16(c128); + combined[1] ^= s2[1].fold_16(c128); + combined[2] ^= s2[2].fold_16(c128); + combined[3] ^= s2[3].fold_16(c128); + combined[4] ^= s2[4].fold_16(c128); + combined[5] ^= s2[5].fold_16(c128); + combined[6] ^= s2[6].fold_16(c128); + combined[7] ^= s2[7].fold_16(c128); + + combined[0] ^= s1[0].fold_16(c256); + combined[1] ^= s1[1].fold_16(c256); + combined[2] ^= s1[2].fold_16(c256); + combined[3] ^= s1[3].fold_16(c256); + combined[4] ^= s1[4].fold_16(c256); + combined[5] ^= s1[5].fold_16(c256); + combined[6] ^= s1[6].fold_16(c256); + combined[7] ^= s1[7].fold_16(c256); + + combined[0] ^= s0[0].fold_16(c384); + combined[1] ^= s0[1].fold_16(c384); + combined[2] ^= s0[2].fold_16(c384); + combined[3] ^= s0[3].fold_16(c384); + combined[4] ^= s0[4].fold_16(c384); + combined[5] ^= s0[5].fold_16(c384); + combined[6] ^= s0[6].fold_16(c384); + combined[7] ^= s0[7].fold_16(c384); + + for block in &blocks[aligned..] { + let b = load_block(block); + fold_block_128(&mut combined, &b, coeff_128); + } - fold_tail(combined, consts) - } // unsafe + fold_tail(combined, consts) } // Public kernels (XZ + NVME) +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc64_vgfm(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 16]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices. from_raw_parts length is exact because - // blocks_u64.len() is a multiple of 16 and tail_u64 is from the aligned middle. +fn crc64_vgfm(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 16]) -> u64 { + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -414,7 +388,7 @@ unsafe fn crc64_vgfm(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, if !tail_u64.is_empty() { // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len() * 8); + let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); state = super::portable::crc64_slice16(state, tail_bytes, tables); } @@ -422,16 +396,19 @@ unsafe fn crc64_vgfm(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, } } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc64_vgfm_2way( +fn crc64_vgfm_2way( mut state: u64, bytes: &[u8], fold_256b: (u64, u64), consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 16], ) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices. from_raw_parts length is exact. + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -448,7 +425,7 @@ unsafe fn crc64_vgfm_2way( if !tail_u64.is_empty() { // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len() * 8); + let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); state = super::portable::crc64_slice16(state, tail_bytes, tables); } @@ -456,8 +433,11 @@ unsafe fn crc64_vgfm_2way( } } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc64_vgfm_4way( +fn crc64_vgfm_4way( mut state: u64, bytes: &[u8], fold_512b: (u64, u64), @@ -465,8 +445,8 @@ unsafe fn crc64_vgfm_4way( consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 16], ) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices. from_raw_parts length is exact. + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -483,7 +463,7 @@ unsafe fn crc64_vgfm_4way( if !tail_u64.is_empty() { // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len() * 8); + let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); state = super::portable::crc64_slice16(state, tail_bytes, tables); } @@ -499,15 +479,12 @@ unsafe fn crc64_vgfm_4way( /// `crate::platform::caps().has(s390x::VECTOR)`. #[target_feature(enable = "vector")] pub(crate) unsafe fn crc64_xz_vgfm(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - unsafe { - crc64_vgfm( - crc, - data, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } + crc64_vgfm( + crc, + data, + &crate::checksum::common::clmul::CRC64_XZ_CLMUL, + &super::kernel_tables::XZ_TABLES_16, + ) } /// CRC-64-XZ using VGFM folding (2-way ILP variant). @@ -518,16 +495,13 @@ pub(crate) unsafe fn crc64_xz_vgfm(crc: u64, data: &[u8]) -> u64 { /// `crate::platform::caps().has(s390x::VECTOR)`. #[target_feature(enable = "vector")] pub(crate) unsafe fn crc64_xz_vgfm_2way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - unsafe { - crc64_vgfm_2way( - crc, - data, - XZ_FOLD_256B, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } + crc64_vgfm_2way( + crc, + data, + XZ_FOLD_256B, + &crate::checksum::common::clmul::CRC64_XZ_CLMUL, + &super::kernel_tables::XZ_TABLES_16, + ) } /// CRC-64-XZ using VGFM folding (4-way ILP variant). @@ -538,17 +512,14 @@ pub(crate) unsafe fn crc64_xz_vgfm_2way(crc: u64, data: &[u8]) -> u64 { /// `crate::platform::caps().has(s390x::VECTOR)`. #[target_feature(enable = "vector")] pub(crate) unsafe fn crc64_xz_vgfm_4way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - unsafe { - crc64_vgfm_4way( - crc, - data, - XZ_FOLD_512B, - &XZ_COMBINE_4WAY, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } + crc64_vgfm_4way( + crc, + data, + XZ_FOLD_512B, + &XZ_COMBINE_4WAY, + &crate::checksum::common::clmul::CRC64_XZ_CLMUL, + &super::kernel_tables::XZ_TABLES_16, + ) } /// CRC-64-NVME using VGFM folding. @@ -559,15 +530,12 @@ pub(crate) unsafe fn crc64_xz_vgfm_4way(crc: u64, data: &[u8]) -> u64 { /// `crate::platform::caps().has(s390x::VECTOR)`. #[target_feature(enable = "vector")] pub(crate) unsafe fn crc64_nvme_vgfm(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - unsafe { - crc64_vgfm( - crc, - data, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } + crc64_vgfm( + crc, + data, + &crate::checksum::common::clmul::CRC64_NVME_CLMUL, + &super::kernel_tables::NVME_TABLES_16, + ) } /// CRC-64-NVME using VGFM folding (2-way ILP variant). @@ -578,16 +546,13 @@ pub(crate) unsafe fn crc64_nvme_vgfm(crc: u64, data: &[u8]) -> u64 { /// `crate::platform::caps().has(s390x::VECTOR)`. #[target_feature(enable = "vector")] pub(crate) unsafe fn crc64_nvme_vgfm_2way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - unsafe { - crc64_vgfm_2way( - crc, - data, - NVME_FOLD_256B, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } + crc64_vgfm_2way( + crc, + data, + NVME_FOLD_256B, + &crate::checksum::common::clmul::CRC64_NVME_CLMUL, + &super::kernel_tables::NVME_TABLES_16, + ) } /// CRC-64-NVME using VGFM folding (4-way ILP variant). @@ -598,53 +563,50 @@ pub(crate) unsafe fn crc64_nvme_vgfm_2way(crc: u64, data: &[u8]) -> u64 { /// `crate::platform::caps().has(s390x::VECTOR)`. #[target_feature(enable = "vector")] pub(crate) unsafe fn crc64_nvme_vgfm_4way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - unsafe { - crc64_vgfm_4way( - crc, - data, - NVME_FOLD_512B, - &NVME_COMBINE_4WAY, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } + crc64_vgfm_4way( + crc, + data, + NVME_FOLD_512B, + &NVME_COMBINE_4WAY, + &crate::checksum::common::clmul::CRC64_NVME_CLMUL, + &super::kernel_tables::NVME_TABLES_16, + ) } // Safe wrappers #[inline] -pub fn crc64_xz_vgfm_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vgfm_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc64_xz_vgfm(crc, data) } } #[inline] -pub fn crc64_xz_vgfm_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vgfm_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc64_xz_vgfm_2way(crc, data) } } #[inline] -pub fn crc64_xz_vgfm_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vgfm_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc64_xz_vgfm_4way(crc, data) } } #[inline] -pub fn crc64_nvme_vgfm_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vgfm_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc64_nvme_vgfm(crc, data) } } #[inline] -pub fn crc64_nvme_vgfm_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vgfm_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc64_nvme_vgfm_2way(crc, data) } } #[inline] -pub fn crc64_nvme_vgfm_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vgfm_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc64_nvme_vgfm_4way(crc, data) } } diff --git a/src/checksum/crc64/x86_64.rs b/src/checksum/crc64/x86_64.rs index 68d625f9..4fb0e0fb 100644 --- a/src/checksum/crc64/x86_64.rs +++ b/src/checksum/crc64/x86_64.rs @@ -7,10 +7,7 @@ //! //! Uses `unsafe` for x86 SIMD intrinsics. Callers must ensure PCLMULQDQ is //! available before executing the accelerated path (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher // SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::x86_64::*, @@ -43,22 +40,33 @@ impl BitXorAssign for Simd { impl Simd { #[inline] #[target_feature(enable = "sse2")] + /// Constructs a SIMD value with `high` above `low`. + /// + /// # Safety + /// + /// The current CPU must support SSE2, which is guaranteed on x86-64. unsafe fn new(high: u64, low: u64) -> Self { - // SAFETY: SSE2 intrinsics are available via this function's #[target_feature] attribute. Self(_mm_set_epi64x(high.cast_signed(), low.cast_signed())) } /// Fold 16 bytes: `(coeff.low ⊗ self.low) ⊕ (coeff.high ⊗ self.high)`. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_16(self, coeff: Self) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. let h = _mm_clmulepi64_si128::<0x11>(self.0, coeff.0); let l = _mm_clmulepi64_si128::<0x00>(self.0, coeff.0); Self(_mm_xor_si128(h, l)) } /// Fold 8 bytes: `self.high ⊕ (coeff ⊗ self.low)`. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_8(self, coeff: u64) -> Self { @@ -70,6 +78,10 @@ impl Simd { } /// Barrett reduction to finalize the CRC. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn barrett(self, poly: u64, mu: u64) -> u64 { @@ -89,6 +101,11 @@ impl Simd { } #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds an initial 128-byte block and its remaining block sequence. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn update_simd(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: &Crc64ClmulConstants) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. unsafe { @@ -125,8 +142,13 @@ unsafe fn update_simd(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: } #[inline(always)] +/// Reduces eight folded SIMD lanes to one CRC-64 state. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. + // SAFETY: The caller's SSE2/PCLMULQDQ guarantee covers the constructor, folds, and reduction. unsafe { // Tail reduction (8×16B → 1×16B), unrolled for throughput. let c0 = Simd::new(consts.tail_fold_16b[0].0, consts.tail_fold_16b[0].1); @@ -154,6 +176,11 @@ unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds one 128-byte block into eight SIMD lanes. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. unsafe { @@ -178,8 +205,13 @@ unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { } #[inline(always)] +/// Folds and XORs one eight-lane stream into an accumulator. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn merge_lanes_xor(acc: &mut [Simd; 8], stream: &[Simd; 8], coeff: Simd) { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. + // SAFETY: The caller's SSE2/PCLMULQDQ guarantee covers all eight lane folds. unsafe { acc[0] ^= stream[0].fold_16(coeff); acc[1] ^= stream[1].fold_16(coeff); @@ -193,6 +225,14 @@ unsafe fn merge_lanes_xor(acc: &mut [Simd; 8], stream: &[Simd; 8], coeff: Simd) } /// 2-way PCLMUL (SSE) kernel with double-unrolling and software prefetch. +/// +/// # Panics +/// +/// Panics if `blocks` contains fewer than two blocks. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn update_simd_2way( state: u64, @@ -229,16 +269,16 @@ unsafe fn update_simd_2way( prefetch_read_l1(prefetch_ptr.wrapping_add(LARGE_BLOCK_DISTANCE)); fold_block_128(&mut s0, &blocks[i], coeff_256); - fold_block_128(&mut s1, &blocks[i + 1], coeff_256); - fold_block_128(&mut s0, &blocks[i + 2], coeff_256); - fold_block_128(&mut s1, &blocks[i + 3], coeff_256); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_256); + fold_block_128(&mut s0, &blocks[i.strict_add(2)], coeff_256); + fold_block_128(&mut s1, &blocks[i.strict_add(3)], coeff_256); i = i.strict_add(4); } // Handle remaining pair. while i < even { fold_block_128(&mut s0, &blocks[i], coeff_256); - fold_block_128(&mut s1, &blocks[i + 1], coeff_256); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_256); i = i.strict_add(2); } @@ -256,6 +296,14 @@ unsafe fn update_simd_2way( } /// 4-way PCLMUL (SSE) kernel with double-unrolling and software prefetch. +/// +/// # Panics +/// +/// Panics in debug builds if `blocks` is empty. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn update_simd_4way( state: u64, @@ -279,7 +327,7 @@ unsafe fn update_simd_4way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 4) * 4; + let aligned = (blocks.len() / 4).strict_mul(4); // Account for starting at i=4: we need i+7 < blocks.len() for valid access let double_aligned = 4usize.strict_add(((blocks.len().strict_sub(4)) / 8).strict_mul(8)); @@ -305,15 +353,15 @@ unsafe fn update_simd_4way( // First group of 4 blocks fold_block_128(&mut s0, &blocks[i], coeff_512); - fold_block_128(&mut s1, &blocks[i + 1], coeff_512); - fold_block_128(&mut s2, &blocks[i + 2], coeff_512); - fold_block_128(&mut s3, &blocks[i + 3], coeff_512); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_512); // Second group of 4 blocks - fold_block_128(&mut s0, &blocks[i + 4], coeff_512); - fold_block_128(&mut s1, &blocks[i + 5], coeff_512); - fold_block_128(&mut s2, &blocks[i + 6], coeff_512); - fold_block_128(&mut s3, &blocks[i + 7], coeff_512); + fold_block_128(&mut s0, &blocks[i.strict_add(4)], coeff_512); + fold_block_128(&mut s1, &blocks[i.strict_add(5)], coeff_512); + fold_block_128(&mut s2, &blocks[i.strict_add(6)], coeff_512); + fold_block_128(&mut s3, &blocks[i.strict_add(7)], coeff_512); i = i.strict_add(8); } @@ -321,9 +369,9 @@ unsafe fn update_simd_4way( // Handle remaining group. while i < aligned { fold_block_128(&mut s0, &blocks[i], coeff_512); - fold_block_128(&mut s1, &blocks[i + 1], coeff_512); - fold_block_128(&mut s2, &blocks[i + 2], coeff_512); - fold_block_128(&mut s3, &blocks[i + 3], coeff_512); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_512); i = i.strict_add(4); } @@ -365,6 +413,14 @@ unsafe fn update_simd_4way( } /// 7-way PCLMUL (SSE) kernel with software prefetch. +/// +/// # Panics +/// +/// Panics in debug builds if `blocks` is empty. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn update_simd_7way( state: u64, @@ -388,7 +444,7 @@ unsafe fn update_simd_7way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 7) * 7; + let aligned = (blocks.len() / 7).strict_mul(7); let coeff_896 = Simd::new(fold_896b.0, fold_896b.1); let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); @@ -415,12 +471,12 @@ unsafe fn update_simd_7way( prefetch_read_l1(prefetch_ptr.wrapping_add(LARGE_BLOCK_DISTANCE)); fold_block_128(&mut s0, &blocks[i], coeff_896); - fold_block_128(&mut s1, &blocks[i + 1], coeff_896); - fold_block_128(&mut s2, &blocks[i + 2], coeff_896); - fold_block_128(&mut s3, &blocks[i + 3], coeff_896); - fold_block_128(&mut s4, &blocks[i + 4], coeff_896); - fold_block_128(&mut s5, &blocks[i + 5], coeff_896); - fold_block_128(&mut s6, &blocks[i + 6], coeff_896); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_896); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_896); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_896); + fold_block_128(&mut s4, &blocks[i.strict_add(4)], coeff_896); + fold_block_128(&mut s5, &blocks[i.strict_add(5)], coeff_896); + fold_block_128(&mut s6, &blocks[i.strict_add(6)], coeff_896); i = i.strict_add(7); } @@ -447,6 +503,14 @@ unsafe fn update_simd_7way( } /// 8-way PCLMUL (SSE) kernel with software prefetch. +/// +/// # Panics +/// +/// Panics in debug builds if `blocks` is empty. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn update_simd_8way( state: u64, @@ -470,7 +534,7 @@ unsafe fn update_simd_8way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 8) * 8; + let aligned = (blocks.len() / 8).strict_mul(8); let coeff_1024 = Simd::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); @@ -498,13 +562,13 @@ unsafe fn update_simd_8way( prefetch_read_l1(prefetch_ptr.wrapping_add(LARGE_BLOCK_DISTANCE)); fold_block_128(&mut s0, &blocks[i], coeff_1024); - fold_block_128(&mut s1, &blocks[i + 1], coeff_1024); - fold_block_128(&mut s2, &blocks[i + 2], coeff_1024); - fold_block_128(&mut s3, &blocks[i + 3], coeff_1024); - fold_block_128(&mut s4, &blocks[i + 4], coeff_1024); - fold_block_128(&mut s5, &blocks[i + 5], coeff_1024); - fold_block_128(&mut s6, &blocks[i + 6], coeff_1024); - fold_block_128(&mut s7, &blocks[i + 7], coeff_1024); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_1024); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_1024); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_1024); + fold_block_128(&mut s4, &blocks[i.strict_add(4)], coeff_1024); + fold_block_128(&mut s5, &blocks[i.strict_add(5)], coeff_1024); + fold_block_128(&mut s6, &blocks[i.strict_add(6)], coeff_1024); + fold_block_128(&mut s7, &blocks[i.strict_add(7)], coeff_1024); i = i.strict_add(8); } @@ -532,9 +596,15 @@ unsafe fn update_simd_8way( } #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds the 128-byte-aligned middle of `bytes` with PCLMULQDQ. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn crc64_pclmul(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 8]) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. - // align_to is sound because Simd is repr(transparent) over __m128i. + // Every bit pattern is valid for `Simd`, which is transparent over `__m128i`, so `align_to` + // produces initialized, non-overlapping fragments. unsafe { let (left, middle, right) = bytes.align_to::<[Simd; 8]>(); if let Some((first, rest)) = middle.split_first() { @@ -548,6 +618,11 @@ unsafe fn crc64_pclmul(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstant } #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds the 128-byte-aligned middle of `bytes` with two PCLMUL streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn crc64_pclmul_2way( mut state: u64, bytes: &[u8], @@ -556,7 +631,8 @@ unsafe fn crc64_pclmul_2way( tables: &[[u64; 256]; 8], ) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. - // align_to is sound because Simd is repr(transparent) over __m128i. + // Every bit pattern is valid for `Simd`, which is transparent over `__m128i`, so `align_to` + // produces initialized, non-overlapping fragments. unsafe { let (left, middle, right) = bytes.align_to::<[Simd; 8]>(); if middle.is_empty() { @@ -576,6 +652,11 @@ unsafe fn crc64_pclmul_2way( } #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds the 128-byte-aligned middle of `bytes` with four PCLMUL streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn crc64_pclmul_4way( mut state: u64, bytes: &[u8], @@ -585,7 +666,8 @@ unsafe fn crc64_pclmul_4way( tables: &[[u64; 256]; 8], ) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. - // align_to is sound because Simd is repr(transparent) over __m128i. + // Every bit pattern is valid for `Simd`, which is transparent over `__m128i`, so `align_to` + // produces initialized, non-overlapping fragments. unsafe { let (left, middle, right) = bytes.align_to::<[Simd; 8]>(); if middle.is_empty() { @@ -599,6 +681,11 @@ unsafe fn crc64_pclmul_4way( } #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds the 128-byte-aligned middle of `bytes` with seven PCLMUL streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn crc64_pclmul_7way( mut state: u64, bytes: &[u8], @@ -608,7 +695,8 @@ unsafe fn crc64_pclmul_7way( tables: &[[u64; 256]; 8], ) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. - // align_to is sound because Simd is repr(transparent) over __m128i. + // Every bit pattern is valid for `Simd`, which is transparent over `__m128i`, so `align_to` + // produces initialized, non-overlapping fragments. unsafe { let (left, middle, right) = bytes.align_to::<[Simd; 8]>(); if middle.is_empty() { @@ -622,6 +710,11 @@ unsafe fn crc64_pclmul_7way( } #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds the 128-byte-aligned middle of `bytes` with eight PCLMUL streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn crc64_pclmul_8way( mut state: u64, bytes: &[u8], @@ -631,7 +724,8 @@ unsafe fn crc64_pclmul_8way( tables: &[[u64; 256]; 8], ) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. - // align_to is sound because Simd is repr(transparent) over __m128i. + // Every bit pattern is valid for `Simd`, which is transparent over `__m128i`, so `align_to` + // produces initialized, non-overlapping fragments. unsafe { let (left, middle, right) = bytes.align_to::<[Simd; 8]>(); if middle.is_empty() { @@ -648,6 +742,10 @@ unsafe fn crc64_pclmul_8way( /// /// This avoids constructing the full 128-byte fold state when only complete /// 16-byte lanes are available. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn crc64_pclmul_small( mut state: u64, @@ -656,7 +754,8 @@ unsafe fn crc64_pclmul_small( tables: &[[u64; 256]; 8], ) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. - // align_to is sound because Simd is repr(transparent) over __m128i. + // Every bit pattern is valid for `Simd`, which is transparent over `__m128i`, so `align_to` + // produces initialized, non-overlapping fragments. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -686,28 +785,20 @@ unsafe fn crc64_pclmul_small( // VPCLMULQDQ (AVX-512) folding -#[inline] -#[target_feature(enable = "avx512f", enable = "vpclmulqdq")] -unsafe fn fold16_4x(x: __m512i, coeff: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. - let h = _mm512_clmulepi64_epi128::<0x11>(x, coeff); - let l = _mm512_clmulepi64_epi128::<0x00>(x, coeff); - _mm512_xor_si512(h, l) -} - /// Fold and XOR with data using VPTERNLOGD (3-way XOR in one instruction). /// /// Computes: `data ^ clmul_hi(x, coeff) ^ clmul_lo(x, coeff)` /// -/// This saves one XOR instruction per fold operation compared to the -/// two-step `data ^ fold16_4x(x, coeff)` pattern. The ternary logic +/// This saves one XOR instruction per fold operation compared with first +/// combining the two products and then XORing the data. The ternary logic /// immediate 0x96 encodes XOR(a, XOR(b, c)) = a ^ b ^ c. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn fold16_4x_ternlog(x: __m512i, data: __m512i, coeff: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. let h = _mm512_clmulepi64_epi128::<0x11>(x, coeff); let l = _mm512_clmulepi64_epi128::<0x00>(x, coeff); // VPTERNLOGD: 3-way XOR (imm8 = 0x96 = a ^ b ^ c) @@ -716,9 +807,12 @@ unsafe fn fold16_4x_ternlog(x: __m512i, data: __m512i, coeff: __m512i) -> __m512 #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Broadcasts a coefficient pair across four 128-bit lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F and VPCLMULQDQ. unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_set_epi64( pair.0.cast_signed(), pair.1.cast_signed(), @@ -733,27 +827,53 @@ unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Loads one 64-byte AVX-512 vector. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. `ptr` must remain valid to read +/// 64 initialized bytes from one allocation. When `ALIGNED` is `true`, `ptr` +/// must also be aligned to 64 bytes; `false` permits any byte alignment. unsafe fn load_m512(ptr: *const u8) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. Caller ensures `ptr` is valid for 64-byte reads (aligned or unaligned as indicated - // by ALIGNED). + // SAFETY: The function attribute enables both intrinsics. A readable 64-byte source cannot be + // null. Casting `NonNull` changes only the pointee type and preserves provenance; the aligned + // specialization's caller additionally guarantees the alignment required by + // `_mm512_load_si512`, while `_mm512_loadu_si512` accepts any byte alignment. unsafe { if ALIGNED { debug_assert_eq!((ptr as usize) & 63, 0); - _mm512_load_si512(ptr.cast::<__m512i>()) + _mm512_load_si512( + core::ptr::NonNull::new_unchecked(ptr.cast_mut()) + .cast::<__m512i>() + .as_ptr(), + ) } else { - _mm512_loadu_si512(ptr.cast::<__m512i>()) + _mm512_loadu_si512( + core::ptr::NonNull::new_unchecked(ptr.cast_mut()) + .cast::<__m512i>() + .as_ptr(), + ) } } } #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Loads one 128-byte block as two adjacent AVX-512 vectors. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. When `ALIGNED` is `true`, +/// `block` must also begin at a 64-byte-aligned address. unsafe fn load_128b_block(block: &[Simd; 8]) -> (__m512i, __m512i) { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. `block` is 128 bytes (8x16B); ptr and ptr+64 are both in-bounds. + // SAFETY: The function attribute enables the intrinsics, and the reference provides 128 + // initialized readable bytes. `from_ref` preserves its provenance; the first load reads bytes + // 0..64 and the second reads bytes 64..128. When `ALIGNED` is true, the caller's 64-byte + // alignment guarantee applies to both addresses. unsafe { - let ptr = block as *const [Simd; 8] as *const u8; + let ptr = core::ptr::from_ref(block).cast::(); // 8×16B lanes packed as 2×64B vectors (4 lanes each). let y0 = load_m512::(ptr); let y1 = load_m512::(ptr.add(64)); @@ -763,9 +883,15 @@ unsafe fn load_128b_block(block: &[Simd; 8]) -> (__m512i, _ #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Reduces two folded AVX-512 vectors to one CRC-64 state. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i, consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The function attribute enables the AVX-512 lane extractions. The caller's complete + // capability guarantee also covers the SSE2/PCLMULQDQ operations used by `fold_tail`. unsafe { // Reuse the well-tested 128-bit tail fold + Barrett reduction by extracting // the 8×16B lanes directly (avoids a store+reload round-trip). @@ -784,11 +910,19 @@ unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i, consts: &Crc64ClmulCo } #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Folds one initial block and its remaining VPCLMUL block sequence. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. If `first` begins at a +/// 64-byte-aligned address, every block in `rest` must also be 64-byte aligned. unsafe fn update_simd_vpclmul(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The caller provides the complete capability set and the alignment relationship + // between `first` and `rest`. `addr` observes only the public address; both specializations + // receive the original references with their provenance unchanged. unsafe { - let aligned = ((first as *const [Simd; 8] as usize) & 63) == 0; + let aligned = (core::ptr::from_ref(first).addr() & 63) == 0; if aligned { update_simd_vpclmul_impl::(state, first, rest, consts) } else { @@ -799,14 +933,22 @@ unsafe fn update_simd_vpclmul(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Folds an initial 128-byte block and its remaining block sequence. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. When `ALIGNED` is `true`, +/// `first` and every block in `rest` must begin at a 64-byte-aligned address. unsafe fn update_simd_vpclmul_impl( state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The function attribute enables the AVX-512 intrinsics, and the caller supplies the + // complete capability set used by the final reduction. The references provide initialized + // 128-byte blocks; when ALIGNED is true, the caller's alignment guarantee covers every load. unsafe { let (mut x0, mut x1) = load_128b_block::(first); @@ -830,9 +972,18 @@ unsafe fn update_simd_vpclmul_impl( } #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Folds the 128-byte middle of `bytes` with VPCLMULQDQ. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. unsafe fn crc64_vpclmul(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 8]) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. align_to is sound because Simd is repr(transparent) over __m128i. + // SAFETY: The function attribute enables the AVX-512 intrinsics, and the caller supplies the + // complete capability set used by the final reduction. Every bit pattern is valid for `Simd`, + // which is transparent over `__m128i`, so `align_to` produces initialized, non-overlapping + // fragments. Middle blocks are 128 bytes apart and therefore share their first block's mod-64 + // alignment, satisfying `update_simd_vpclmul`'s specialization contract. unsafe { let (left, middle, right) = bytes.align_to::<[Simd; 8]>(); if let Some((first, rest)) = middle.split_first() { @@ -848,16 +999,23 @@ unsafe fn crc64_vpclmul(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstan // VPCLMULQDQ multi-stream (2/4/7-way, 128B blocks) #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Selects the aligned or unaligned two-stream VPCLMUL fold. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. unsafe fn update_simd_vpclmul_2way( state: u64, blocks: &[[Simd; 8]], fold_256b: (u64, u64), consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The caller supplies the complete capability set. `addr` observes only the public + // address, and 128-byte array-element strides preserve its mod-64 alignment for every block; + // both specializations receive the original slice with its provenance unchanged. unsafe { - let aligned = ((blocks.as_ptr() as usize) & 63) == 0; + let aligned = (blocks.as_ptr().addr() & 63) == 0; if aligned { update_simd_vpclmul_2way_impl::(state, blocks, fold_256b, consts) } else { @@ -869,6 +1027,12 @@ unsafe fn update_simd_vpclmul_2way( /// 2-way VPCLMUL kernel with double-unrolling and software prefetch. /// /// Processes 512 bytes per iteration (4 × 128B blocks in 2 streams). +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. When `ALIGNED` is `true`, +/// every block in `blocks` must begin at a 64-byte-aligned address. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn update_simd_vpclmul_2way_impl( state: u64, @@ -876,9 +1040,10 @@ unsafe fn update_simd_vpclmul_2way_impl( fold_256b: (u64, u64), consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. Pointer arithmetic on `blocks.as_ptr()` is in-bounds: indices are bounded by - // `even`/`double_even` < `blocks.len()`. + // SAFETY: The caller supplies the complete capability set and the alignment required by the + // selected loads. After the fallback, len >= 2. `double_even` is 2 + 4k and <= len, so each + // double-loop i + {0,1,2,3} is in-bounds; i < even makes each remaining-pair i + {0,1} + // in-bounds; and even < len guards the odd tail. The prefetch base uses the in-bounds i. unsafe { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; @@ -918,16 +1083,16 @@ unsafe fn update_simd_vpclmul_2way_impl( x0_0 = fold16_4x_ternlog(x0_0, y0, coeff_256); x1_0 = fold16_4x_ternlog(x1_0, y1, coeff_256); - let (y0, y1) = load_128b_block::(&blocks[i + 1]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(1)]); x0_1 = fold16_4x_ternlog(x0_1, y0, coeff_256); x1_1 = fold16_4x_ternlog(x1_1, y1, coeff_256); // Second pair of blocks (256B) - let (z0, z1) = load_128b_block::(&blocks[i + 2]); + let (z0, z1) = load_128b_block::(&blocks[i.strict_add(2)]); x0_0 = fold16_4x_ternlog(x0_0, z0, coeff_256); x1_0 = fold16_4x_ternlog(x1_0, z1, coeff_256); - let (z0, z1) = load_128b_block::(&blocks[i + 3]); + let (z0, z1) = load_128b_block::(&blocks[i.strict_add(3)]); x0_1 = fold16_4x_ternlog(x0_1, z0, coeff_256); x1_1 = fold16_4x_ternlog(x1_1, z1, coeff_256); @@ -940,7 +1105,7 @@ unsafe fn update_simd_vpclmul_2way_impl( x0_0 = fold16_4x_ternlog(x0_0, y0, coeff_256); x1_0 = fold16_4x_ternlog(x1_0, y1, coeff_256); - let (y0, y1) = load_128b_block::(&blocks[i + 1]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(1)]); x0_1 = fold16_4x_ternlog(x0_1, y0, coeff_256); x1_1 = fold16_4x_ternlog(x1_1, y1, coeff_256); @@ -963,6 +1128,12 @@ unsafe fn update_simd_vpclmul_2way_impl( } #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Selects the aligned or unaligned four-stream VPCLMUL fold. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. unsafe fn update_simd_vpclmul_4way( state: u64, blocks: &[[Simd; 8]], @@ -970,10 +1141,11 @@ unsafe fn update_simd_vpclmul_4way( combine: &[(u64, u64); 3], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The caller supplies the complete capability set. `addr` observes only the public + // address, and 128-byte array-element strides preserve its mod-64 alignment for every block; + // both specializations receive the original slice with its provenance unchanged. unsafe { - let aligned = ((blocks.as_ptr() as usize) & 63) == 0; + let aligned = (blocks.as_ptr().addr() & 63) == 0; if aligned { update_simd_vpclmul_4way_impl::(state, blocks, fold_512b, combine, consts) } else { @@ -985,6 +1157,12 @@ unsafe fn update_simd_vpclmul_4way( /// 4-way VPCLMUL kernel with double-unrolling and software prefetch. /// /// Processes 1024 bytes per iteration (8 × 128B blocks in 4 streams). +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. When `ALIGNED` is `true`, +/// every block in `blocks` must begin at a 64-byte-aligned address. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn update_simd_vpclmul_4way_impl( state: u64, @@ -993,9 +1171,10 @@ unsafe fn update_simd_vpclmul_4way_impl( combine: &[(u64, u64); 3], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. Pointer arithmetic on `blocks.as_ptr()` is in-bounds: indices are bounded by - // `aligned`/`double_aligned` < `blocks.len()`. + // SAFETY: The caller supplies the complete capability set and the alignment required by the + // selected loads. After the fallback, len >= 4. `double_aligned` is 4 + 8k and <= len, so each + // double-loop i + {0..7} is in-bounds; i < aligned makes each remaining-group i + {0..3} + // in-bounds; and aligned <= len makes the final suffix valid. The prefetch base uses in-bounds i. unsafe { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; @@ -1008,7 +1187,7 @@ unsafe fn update_simd_vpclmul_4way_impl( return update_simd_vpclmul(state, first, rest, consts); } - let aligned = (blocks.len() / 4) * 4; + let aligned = (blocks.len() / 4).strict_mul(4); // Account for starting at i=4: we need i+7 < blocks.len() for valid access let double_aligned = 4usize.strict_add(((blocks.len().strict_sub(4)) / 8).strict_mul(8)); @@ -1038,32 +1217,32 @@ unsafe fn update_simd_vpclmul_4way_impl( x0_0 = fold16_4x_ternlog(x0_0, y0, coeff_512); x1_0 = fold16_4x_ternlog(x1_0, y1, coeff_512); - let (y0, y1) = load_128b_block::(&blocks[i + 1]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(1)]); x0_1 = fold16_4x_ternlog(x0_1, y0, coeff_512); x1_1 = fold16_4x_ternlog(x1_1, y1, coeff_512); - let (y0, y1) = load_128b_block::(&blocks[i + 2]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(2)]); x0_2 = fold16_4x_ternlog(x0_2, y0, coeff_512); x1_2 = fold16_4x_ternlog(x1_2, y1, coeff_512); - let (y0, y1) = load_128b_block::(&blocks[i + 3]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(3)]); x0_3 = fold16_4x_ternlog(x0_3, y0, coeff_512); x1_3 = fold16_4x_ternlog(x1_3, y1, coeff_512); // Second group of 4 blocks (512B) - let (z0, z1) = load_128b_block::(&blocks[i + 4]); + let (z0, z1) = load_128b_block::(&blocks[i.strict_add(4)]); x0_0 = fold16_4x_ternlog(x0_0, z0, coeff_512); x1_0 = fold16_4x_ternlog(x1_0, z1, coeff_512); - let (z0, z1) = load_128b_block::(&blocks[i + 5]); + let (z0, z1) = load_128b_block::(&blocks[i.strict_add(5)]); x0_1 = fold16_4x_ternlog(x0_1, z0, coeff_512); x1_1 = fold16_4x_ternlog(x1_1, z1, coeff_512); - let (z0, z1) = load_128b_block::(&blocks[i + 6]); + let (z0, z1) = load_128b_block::(&blocks[i.strict_add(6)]); x0_2 = fold16_4x_ternlog(x0_2, z0, coeff_512); x1_2 = fold16_4x_ternlog(x1_2, z1, coeff_512); - let (z0, z1) = load_128b_block::(&blocks[i + 7]); + let (z0, z1) = load_128b_block::(&blocks[i.strict_add(7)]); x0_3 = fold16_4x_ternlog(x0_3, z0, coeff_512); x1_3 = fold16_4x_ternlog(x1_3, z1, coeff_512); @@ -1076,15 +1255,15 @@ unsafe fn update_simd_vpclmul_4way_impl( x0_0 = fold16_4x_ternlog(x0_0, y0, coeff_512); x1_0 = fold16_4x_ternlog(x1_0, y1, coeff_512); - let (y0, y1) = load_128b_block::(&blocks[i + 1]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(1)]); x0_1 = fold16_4x_ternlog(x0_1, y0, coeff_512); x1_1 = fold16_4x_ternlog(x1_1, y1, coeff_512); - let (y0, y1) = load_128b_block::(&blocks[i + 2]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(2)]); x0_2 = fold16_4x_ternlog(x0_2, y0, coeff_512); x1_2 = fold16_4x_ternlog(x1_2, y1, coeff_512); - let (y0, y1) = load_128b_block::(&blocks[i + 3]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(3)]); x0_3 = fold16_4x_ternlog(x0_3, y0, coeff_512); x1_3 = fold16_4x_ternlog(x1_3, y1, coeff_512); @@ -1111,6 +1290,12 @@ unsafe fn update_simd_vpclmul_4way_impl( } #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Selects the aligned or unaligned seven-stream VPCLMUL fold. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. unsafe fn update_simd_vpclmul_7way( state: u64, blocks: &[[Simd; 8]], @@ -1118,10 +1303,11 @@ unsafe fn update_simd_vpclmul_7way( combine: &[(u64, u64); 6], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The caller supplies the complete capability set. `addr` observes only the public + // address, and 128-byte array-element strides preserve its mod-64 alignment for every block; + // both specializations receive the original slice with its provenance unchanged. unsafe { - let aligned = ((blocks.as_ptr() as usize) & 63) == 0; + let aligned = (blocks.as_ptr().addr() & 63) == 0; if aligned { update_simd_vpclmul_7way_impl::(state, blocks, fold_896b, combine, consts) } else { @@ -1134,6 +1320,12 @@ unsafe fn update_simd_vpclmul_7way( /// /// Processes 896 bytes per iteration (7 × 128B blocks in 7 streams). /// Already has high ILP from 7-way parallelism; prefetch helps hide memory latency. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. When `ALIGNED` is `true`, +/// every block in `blocks` must begin at a 64-byte-aligned address. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn update_simd_vpclmul_7way_impl( state: u64, @@ -1142,9 +1334,10 @@ unsafe fn update_simd_vpclmul_7way_impl( combine: &[(u64, u64); 6], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. Pointer arithmetic on `blocks.as_ptr()` is in-bounds: indices are bounded by - // `aligned` < `blocks.len()`. + // SAFETY: The caller supplies the complete capability set and the alignment required by the + // selected loads. After the fallback, len >= 7. `aligned` is a multiple of 7 and <= len; i + // starts at 7 and advances by 7, so i < aligned makes every i + {0..6} and the prefetch base + // in-bounds. `aligned <= len` also makes the final suffix valid. unsafe { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; @@ -1157,7 +1350,7 @@ unsafe fn update_simd_vpclmul_7way_impl( return update_simd_vpclmul(state, first, rest, consts); } - let aligned = (blocks.len() / 7) * 7; + let aligned = (blocks.len() / 7).strict_mul(7); let (mut x0_0, mut x1_0) = load_128b_block::(&blocks[0]); let (mut x0_1, mut x1_1) = load_128b_block::(&blocks[1]); @@ -1189,27 +1382,27 @@ unsafe fn update_simd_vpclmul_7way_impl( x0_0 = fold16_4x_ternlog(x0_0, y0, coeff_896); x1_0 = fold16_4x_ternlog(x1_0, y1, coeff_896); - let (y0, y1) = load_128b_block::(&blocks[i + 1]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(1)]); x0_1 = fold16_4x_ternlog(x0_1, y0, coeff_896); x1_1 = fold16_4x_ternlog(x1_1, y1, coeff_896); - let (y0, y1) = load_128b_block::(&blocks[i + 2]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(2)]); x0_2 = fold16_4x_ternlog(x0_2, y0, coeff_896); x1_2 = fold16_4x_ternlog(x1_2, y1, coeff_896); - let (y0, y1) = load_128b_block::(&blocks[i + 3]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(3)]); x0_3 = fold16_4x_ternlog(x0_3, y0, coeff_896); x1_3 = fold16_4x_ternlog(x1_3, y1, coeff_896); - let (y0, y1) = load_128b_block::(&blocks[i + 4]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(4)]); x0_4 = fold16_4x_ternlog(x0_4, y0, coeff_896); x1_4 = fold16_4x_ternlog(x1_4, y1, coeff_896); - let (y0, y1) = load_128b_block::(&blocks[i + 5]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(5)]); x0_5 = fold16_4x_ternlog(x0_5, y0, coeff_896); x1_5 = fold16_4x_ternlog(x1_5, y1, coeff_896); - let (y0, y1) = load_128b_block::(&blocks[i + 6]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(6)]); x0_6 = fold16_4x_ternlog(x0_6, y0, coeff_896); x1_6 = fold16_4x_ternlog(x1_6, y1, coeff_896); @@ -1245,6 +1438,12 @@ unsafe fn update_simd_vpclmul_7way_impl( } #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Selects the aligned or unaligned eight-stream VPCLMUL fold. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. unsafe fn update_simd_vpclmul_8way( state: u64, blocks: &[[Simd; 8]], @@ -1252,10 +1451,11 @@ unsafe fn update_simd_vpclmul_8way( combine: &[(u64, u64); 7], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The caller supplies the complete capability set. `addr` observes only the public + // address, and 128-byte array-element strides preserve its mod-64 alignment for every block; + // both specializations receive the original slice with its provenance unchanged. unsafe { - let aligned = ((blocks.as_ptr() as usize) & 63) == 0; + let aligned = (blocks.as_ptr().addr() & 63) == 0; if aligned { update_simd_vpclmul_8way_impl::(state, blocks, fold_1024b, combine, consts) } else { @@ -1268,6 +1468,12 @@ unsafe fn update_simd_vpclmul_8way( /// /// Processes 1024 bytes per iteration (8 × 128B blocks in 8 streams). /// Already has high ILP from 8-way parallelism; prefetch helps hide memory latency. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. When `ALIGNED` is `true`, +/// every block in `blocks` must begin at a 64-byte-aligned address. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn update_simd_vpclmul_8way_impl( state: u64, @@ -1276,9 +1482,10 @@ unsafe fn update_simd_vpclmul_8way_impl( combine: &[(u64, u64); 7], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. Pointer arithmetic on `blocks.as_ptr()` is in-bounds: indices are bounded by - // `aligned` < `blocks.len()`. + // SAFETY: The caller supplies the complete capability set and the alignment required by the + // selected loads. After the fallback, len >= 8. `aligned` is a multiple of 8 and <= len; i + // starts at 8 and advances by 8, so i < aligned makes every i + {0..7} and the prefetch base + // in-bounds. `aligned <= len` also makes the final suffix valid. unsafe { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; @@ -1291,7 +1498,7 @@ unsafe fn update_simd_vpclmul_8way_impl( return update_simd_vpclmul(state, first, rest, consts); } - let aligned = (blocks.len() / 8) * 8; + let aligned = (blocks.len() / 8).strict_mul(8); let (mut x0_0, mut x1_0) = load_128b_block::(&blocks[0]); let (mut x0_1, mut x1_1) = load_128b_block::(&blocks[1]); @@ -1324,31 +1531,31 @@ unsafe fn update_simd_vpclmul_8way_impl( x0_0 = fold16_4x_ternlog(x0_0, y0, coeff_1024); x1_0 = fold16_4x_ternlog(x1_0, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 1]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(1)]); x0_1 = fold16_4x_ternlog(x0_1, y0, coeff_1024); x1_1 = fold16_4x_ternlog(x1_1, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 2]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(2)]); x0_2 = fold16_4x_ternlog(x0_2, y0, coeff_1024); x1_2 = fold16_4x_ternlog(x1_2, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 3]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(3)]); x0_3 = fold16_4x_ternlog(x0_3, y0, coeff_1024); x1_3 = fold16_4x_ternlog(x1_3, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 4]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(4)]); x0_4 = fold16_4x_ternlog(x0_4, y0, coeff_1024); x1_4 = fold16_4x_ternlog(x1_4, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 5]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(5)]); x0_5 = fold16_4x_ternlog(x0_5, y0, coeff_1024); x1_5 = fold16_4x_ternlog(x1_5, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 6]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(6)]); x0_6 = fold16_4x_ternlog(x0_6, y0, coeff_1024); x1_6 = fold16_4x_ternlog(x1_6, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 7]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(7)]); x0_7 = fold16_4x_ternlog(x0_7, y0, coeff_1024); x1_7 = fold16_4x_ternlog(x1_7, y1, coeff_1024); @@ -1397,6 +1604,11 @@ unsafe fn update_simd_vpclmul_8way_impl( /// 1. Fold x0,x1 into x2,x3 (1024-bit shift = 128 bytes) /// 2. Fold result into single 512-bit register (512-bit shift = 64 bytes) /// 3. Extract 4×128-bit lanes and reduce to u64 +/// +/// # Safety +/// +/// Requires the x86 features represented by `crate::platform::caps::x86::VPCLMUL_READY`. +/// The caller must verify that capability set before calling this function. #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn finalize_4x512_state( @@ -1448,6 +1660,11 @@ unsafe fn finalize_4x512_state( /// /// The double-unroll halves the number of loop-control steps. Software /// prefetch remains a target-dependent hint. +/// +/// # Safety +/// +/// Requires the x86 features represented by `crate::platform::caps::x86::VPCLMUL_READY`. +/// The caller must verify that capability set before calling this function. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn crc64_vpclmul_4x512( mut state: u64, @@ -1456,10 +1673,11 @@ unsafe fn crc64_vpclmul_4x512( consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 8], ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. align_to is sound because Simd is repr(transparent) over __m128i. - // Pointer arithmetic: `ptr` stays within `aligned_bytes`; bounds checked via `ptr.add(...) <= end`. - // from_raw_parts: `ptr` is valid for `remaining` bytes (derived from `end.offset_from(ptr)`). + // SAFETY: The caller provides the full VPCLMUL_READY capability set. `ptr` and `end` retain the + // provenance and order of the same slice allocation. The initial unsigned-distance check makes + // `last_double_block` in-bounds, and every aligned 64-byte load and pointer advance is dominated + // by that bound or a matching tail-distance check. The final pointer and tail length therefore + // describe initialized bytes from that allocation. unsafe { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; @@ -1493,10 +1711,10 @@ unsafe fn crc64_vpclmul_4x512( debug_assert_eq!((ptr as usize) & 63, 0); // Load first 256B block into 4 __m512i registers. - let mut x0 = _mm512_load_si512(ptr.cast::<__m512i>()); - let mut x1 = _mm512_load_si512(ptr.add(64).cast::<__m512i>()); - let mut x2 = _mm512_load_si512(ptr.add(128).cast::<__m512i>()); - let mut x3 = _mm512_load_si512(ptr.add(192).cast::<__m512i>()); + let mut x0 = load_m512::(ptr); + let mut x1 = load_m512::(ptr.add(64)); + let mut x2 = load_m512::(ptr.add(128)); + let mut x3 = load_m512::(ptr.add(192)); ptr = ptr.add(BLOCK_SIZE); // XOR the initial CRC into lane 0 (low 64 bits of first register). @@ -1508,41 +1726,44 @@ unsafe fn crc64_vpclmul_4x512( // Double-unrolled main loop: fold 512B (2 blocks) per iteration. // This reduces loop overhead by 50% and improves ILP. - while ptr.add(DOUBLE_BLOCK) <= end { - // Prefetch 2 iterations ahead (1KB) to hide memory latency. - prefetch_read_l1(ptr.wrapping_add(LARGE_BLOCK_DISTANCE)); - - // First block (256B): load and fold - let y0 = _mm512_load_si512(ptr.cast::<__m512i>()); - let y1 = _mm512_load_si512(ptr.add(64).cast::<__m512i>()); - let y2 = _mm512_load_si512(ptr.add(128).cast::<__m512i>()); - let y3 = _mm512_load_si512(ptr.add(192).cast::<__m512i>()); - - x0 = fold16_4x_ternlog(x0, y0, coeff_256); - x1 = fold16_4x_ternlog(x1, y1, coeff_256); - x2 = fold16_4x_ternlog(x2, y2, coeff_256); - x3 = fold16_4x_ternlog(x3, y3, coeff_256); - - // Second block (256B): load and fold - let z0 = _mm512_load_si512(ptr.add(256).cast::<__m512i>()); - let z1 = _mm512_load_si512(ptr.add(320).cast::<__m512i>()); - let z2 = _mm512_load_si512(ptr.add(384).cast::<__m512i>()); - let z3 = _mm512_load_si512(ptr.add(448).cast::<__m512i>()); - - x0 = fold16_4x_ternlog(x0, z0, coeff_256); - x1 = fold16_4x_ternlog(x1, z1, coeff_256); - x2 = fold16_4x_ternlog(x2, z2, coeff_256); - x3 = fold16_4x_ternlog(x3, z3, coeff_256); - - ptr = ptr.add(DOUBLE_BLOCK); + if end.offset_from_unsigned(ptr) >= DOUBLE_BLOCK { + let last_double_block = end.sub(DOUBLE_BLOCK); + while ptr <= last_double_block { + // Prefetch 2 iterations ahead (1KB) to hide memory latency. + prefetch_read_l1(ptr.wrapping_add(LARGE_BLOCK_DISTANCE)); + + // First block (256B): load and fold + let y0 = load_m512::(ptr); + let y1 = load_m512::(ptr.add(64)); + let y2 = load_m512::(ptr.add(128)); + let y3 = load_m512::(ptr.add(192)); + + x0 = fold16_4x_ternlog(x0, y0, coeff_256); + x1 = fold16_4x_ternlog(x1, y1, coeff_256); + x2 = fold16_4x_ternlog(x2, y2, coeff_256); + x3 = fold16_4x_ternlog(x3, y3, coeff_256); + + // Second block (256B): load and fold + let z0 = load_m512::(ptr.add(256)); + let z1 = load_m512::(ptr.add(320)); + let z2 = load_m512::(ptr.add(384)); + let z3 = load_m512::(ptr.add(448)); + + x0 = fold16_4x_ternlog(x0, z0, coeff_256); + x1 = fold16_4x_ternlog(x1, z1, coeff_256); + x2 = fold16_4x_ternlog(x2, z2, coeff_256); + x3 = fold16_4x_ternlog(x3, z3, coeff_256); + + ptr = ptr.add(DOUBLE_BLOCK); + } } // Handle remaining single block (if odd number of blocks). - if ptr.add(BLOCK_SIZE) <= end { - let y0 = _mm512_load_si512(ptr.cast::<__m512i>()); - let y1 = _mm512_load_si512(ptr.add(64).cast::<__m512i>()); - let y2 = _mm512_load_si512(ptr.add(128).cast::<__m512i>()); - let y3 = _mm512_load_si512(ptr.add(192).cast::<__m512i>()); + if end.offset_from_unsigned(ptr) >= BLOCK_SIZE { + let y0 = load_m512::(ptr); + let y1 = load_m512::(ptr.add(64)); + let y2 = load_m512::(ptr.add(128)); + let y3 = load_m512::(ptr.add(192)); x0 = fold16_4x_ternlog(x0, y0, coeff_256); x1 = fold16_4x_ternlog(x1, y1, coeff_256); @@ -1556,11 +1777,17 @@ unsafe fn crc64_vpclmul_4x512( state = finalize_4x512_state(x0, x1, x2, x3, consts.fold_128b, consts.tail_fold_16b[3], consts); // Process any remaining bytes. - let remaining = end.offset_from(ptr) as usize; + let remaining = end.offset_from_unsigned(ptr); super::portable::crc64_slice8(state, core::slice::from_raw_parts(ptr, remaining), tables) } } +/// Fold CRC-64 with two parallel VPCLMUL streams. +/// +/// # Safety +/// +/// Requires the x86 features represented by `crate::platform::caps::x86::VPCLMUL_READY`. +/// The caller must verify that capability set before calling this function. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn crc64_vpclmul_2way( mut state: u64, @@ -1583,6 +1810,12 @@ unsafe fn crc64_vpclmul_2way( } } +/// Fold CRC-64 with four parallel VPCLMUL streams. +/// +/// # Safety +/// +/// Requires the x86 features represented by `crate::platform::caps::x86::VPCLMUL_READY`. +/// The caller must verify that capability set before calling this function. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn crc64_vpclmul_4way( mut state: u64, @@ -1606,6 +1839,12 @@ unsafe fn crc64_vpclmul_4way( } } +/// Fold CRC-64 with seven parallel VPCLMUL streams. +/// +/// # Safety +/// +/// Requires the x86 features represented by `crate::platform::caps::x86::VPCLMUL_READY`. +/// The caller must verify that capability set before calling this function. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn crc64_vpclmul_7way( mut state: u64, @@ -1629,6 +1868,12 @@ unsafe fn crc64_vpclmul_7way( } } +/// Fold CRC-64 with eight parallel VPCLMUL streams. +/// +/// # Safety +/// +/// Requires the x86 features represented by `crate::platform::caps::x86::VPCLMUL_READY`. +/// The caller must verify that capability set before calling this function. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn crc64_vpclmul_8way( mut state: u64, @@ -2131,199 +2376,169 @@ pub(crate) unsafe fn crc64_nvme_vpclmul_8way(crc: u64, data: &[u8]) -> u64 { } } -/// CRC-64-NVME using VPCLMULQDQ (4×512-bit variant). -/// -/// Processes 512 bytes per main-loop iteration across four fold streams. -/// -/// # Safety -/// -/// Requires VPCLMULQDQ + AVX-512. Caller must verify via -/// `crate::platform::caps().has(x86::VPCLMUL_READY)`. -#[target_feature(enable = "avx512f", enable = "vpclmulqdq")] -pub(crate) unsafe fn crc64_nvme_vpclmul_4x512(crc: u64, data: &[u8]) -> u64 { - // SAFETY: This wrapper relies on the function's safety contract (caller ensures VPCLMULQDQ/AVX-512 - // is available). - unsafe { - crc64_vpclmul_4x512( - crc, - data, - CRC64_NVME_STREAM.fold_256b, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_8, - ) - } -} - // Dispatcher Wrappers (safe interface) /// Safe wrapper for CRC-64-XZ PCLMUL kernel. #[inline] -pub fn crc64_xz_pclmul_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pclmul_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_xz_pclmul(crc, data) } } /// Safe wrapper for CRC-64-XZ PCLMUL small-buffer kernel. #[inline] -pub fn crc64_xz_pclmul_small_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pclmul_small_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_xz_pclmul_small(crc, data) } } /// Safe wrapper for CRC-64-XZ PCLMUL 2-way kernel. #[inline] -pub fn crc64_xz_pclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_xz_pclmul_2way(crc, data) } } /// Safe wrapper for CRC-64-XZ PCLMUL 4-way kernel. #[inline] -pub fn crc64_xz_pclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_xz_pclmul_4way(crc, data) } } /// Safe wrapper for CRC-64-XZ PCLMUL 7-way kernel. #[inline] -pub fn crc64_xz_pclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_xz_pclmul_7way(crc, data) } } /// Safe wrapper for CRC-64-XZ PCLMUL 8-way kernel. #[inline] -pub fn crc64_xz_pclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_xz_pclmul_8way(crc, data) } } /// Safe wrapper for CRC-64-XZ VPCLMUL kernel. #[inline] -pub fn crc64_xz_vpclmul_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpclmul_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc64_xz_vpclmul(crc, data) } } /// Safe wrapper for CRC-64-XZ VPCLMUL 2-way kernel. #[inline] -pub fn crc64_xz_vpclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_xz_vpclmul_2way(crc, data) } } /// Safe wrapper for CRC-64-XZ VPCLMUL 4-way kernel. #[inline] -pub fn crc64_xz_vpclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_xz_vpclmul_4way(crc, data) } } /// Safe wrapper for CRC-64-XZ VPCLMUL 7-way kernel. #[inline] -pub fn crc64_xz_vpclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_xz_vpclmul_7way(crc, data) } } /// Safe wrapper for CRC-64-XZ VPCLMUL 8-way kernel. #[inline] -pub fn crc64_xz_vpclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_xz_vpclmul_8way(crc, data) } } /// Safe wrapper for CRC-64-XZ VPCLMUL 4×512-bit kernel. #[inline] -pub fn crc64_xz_vpclmul_4x512_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpclmul_4x512_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_xz_vpclmul_4x512(crc, data) } } /// Safe wrapper for CRC-64-NVME PCLMUL kernel. #[inline] -pub fn crc64_nvme_pclmul_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pclmul_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_nvme_pclmul(crc, data) } } /// Safe wrapper for CRC-64-NVME PCLMUL small-buffer kernel. #[inline] -pub fn crc64_nvme_pclmul_small_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pclmul_small_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_nvme_pclmul_small(crc, data) } } /// Safe wrapper for CRC-64-NVME PCLMUL 2-way kernel. #[inline] -pub fn crc64_nvme_pclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_nvme_pclmul_2way(crc, data) } } /// Safe wrapper for CRC-64-NVME PCLMUL 4-way kernel. #[inline] -pub fn crc64_nvme_pclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_nvme_pclmul_4way(crc, data) } } /// Safe wrapper for CRC-64-NVME PCLMUL 7-way kernel. #[inline] -pub fn crc64_nvme_pclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_nvme_pclmul_7way(crc, data) } } /// Safe wrapper for CRC-64-NVME PCLMUL 8-way kernel. #[inline] -pub fn crc64_nvme_pclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_nvme_pclmul_8way(crc, data) } } /// Safe wrapper for CRC-64-NVME VPCLMUL kernel. #[inline] -pub fn crc64_nvme_vpclmul_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpclmul_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc64_nvme_vpclmul(crc, data) } } /// Safe wrapper for CRC-64-NVME VPCLMUL 2-way kernel. #[inline] -pub fn crc64_nvme_vpclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_nvme_vpclmul_2way(crc, data) } } /// Safe wrapper for CRC-64-NVME VPCLMUL 4-way kernel. #[inline] -pub fn crc64_nvme_vpclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_nvme_vpclmul_4way(crc, data) } } /// Safe wrapper for CRC-64-NVME VPCLMUL 7-way kernel. #[inline] -pub fn crc64_nvme_vpclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_nvme_vpclmul_7way(crc, data) } } /// Safe wrapper for CRC-64-NVME VPCLMUL 8-way kernel. #[inline] -pub fn crc64_nvme_vpclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_nvme_vpclmul_8way(crc, data) } } -/// Safe wrapper for CRC-64-NVME VPCLMUL 4×512-bit kernel. -#[inline] -pub fn crc64_nvme_vpclmul_4x512_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. - unsafe { crc64_nvme_vpclmul_4x512(crc, data) } -} - // Tests // Tests require SIMD intrinsics that Miri cannot interpret. @@ -2336,8 +2551,17 @@ mod tests { use super::*; fn make_data(len: usize) -> Vec { + let mut byte_index = 0u8; + let mut group_index = 0u8; (0..len) - .map(|i| (i as u8).wrapping_mul(17).wrapping_add((i >> 3) as u8)) + .map(|_| { + let byte = byte_index.wrapping_mul(17).wrapping_add(group_index); + byte_index = byte_index.wrapping_add(1); + if byte_index & 7 == 0 { + group_index = group_index.wrapping_add(1); + } + byte + }) .collect() } @@ -2485,37 +2709,108 @@ mod tests { #[test] fn test_crc64_xz_vpclmul_multiway_matches_portable_various_lengths() { - if !(std::arch::is_x86_feature_detected!("avx512f") && std::arch::is_x86_feature_detected!("vpclmulqdq")) { + if !crate::platform::caps().has(crate::platform::caps::x86::VPCLMUL_READY) { return; } - for len in [0usize, 1, 7, 16, 63, 64, 127, 128, 255, 256, 512, 1024, 4096, 16 * 1024] { + for len in [ + 0usize, + 1, + 7, + 16, + 63, + 64, + 127, + 128, + 255, + 256, + 512, + 1023, + 1024, + 1025, + 4096, + 16 * 1024, + ] { let data = make_data(len); let portable = super::super::portable::crc64_slice8_xz(!0, &data) ^ !0; let vp2 = crc64_xz_vpclmul_2way_safe(!0, &data) ^ !0; let vp4 = crc64_xz_vpclmul_4way_safe(!0, &data) ^ !0; let vp7 = crc64_xz_vpclmul_7way_safe(!0, &data) ^ !0; + let vp8 = crc64_xz_vpclmul_8way_safe(!0, &data) ^ !0; assert_eq!(vp2, portable, "2-way mismatch at len={len}"); assert_eq!(vp4, portable, "4-way mismatch at len={len}"); assert_eq!(vp7, portable, "7-way mismatch at len={len}"); + assert_eq!(vp8, portable, "8-way mismatch at len={len}"); + } + } + + #[test] + fn test_crc64_xz_vpclmul_4x512_matches_portable_at_alignment_boundaries() { + if !crate::platform::caps().has(crate::platform::caps::x86::VPCLMUL_READY) { + return; + } + + const BUFFER_LEN: usize = 8256; + const LENGTHS: &[usize] = &[ + 0, 255, 256, 257, 511, 512, 513, 767, 768, 769, 1023, 1024, 1025, 4096, 4097, 8193, + ]; + const STATES: &[u64] = &[0, u64::MAX, 0x0123_4567_89AB_CDEF, 0xDEAD_BEEF_CAFE_BABE]; + + #[repr(align(64))] + struct AlignedBytes([u8; BUFFER_LEN]); + + let mut data = AlignedBytes([0; BUFFER_LEN]); + data.0.copy_from_slice(&make_data(BUFFER_LEN)); + + for &state in STATES { + for offset in 0usize..64 { + for &len in LENGTHS { + let input = &data.0[offset..offset.strict_add(len)]; + let expected_xz = super::super::portable::crc64_slice8_xz(state, input); + assert_eq!( + crc64_xz_vpclmul_4x512_safe(state, input), + expected_xz, + "XZ mismatch at state={state:#018x}, offset={offset}, len={len}" + ); + } + } } } #[test] fn test_crc64_nvme_vpclmul_multiway_matches_portable_various_lengths() { - if !(std::arch::is_x86_feature_detected!("avx512f") && std::arch::is_x86_feature_detected!("vpclmulqdq")) { + if !crate::platform::caps().has(crate::platform::caps::x86::VPCLMUL_READY) { return; } - for len in [0usize, 1, 7, 16, 63, 64, 127, 128, 255, 256, 512, 1024, 4096, 16 * 1024] { + for len in [ + 0usize, + 1, + 7, + 16, + 63, + 64, + 127, + 128, + 255, + 256, + 512, + 1023, + 1024, + 1025, + 4096, + 16 * 1024, + ] { let data = make_data(len); let portable = super::super::portable::crc64_slice8_nvme(!0, &data) ^ !0; let vp2 = crc64_nvme_vpclmul_2way_safe(!0, &data) ^ !0; let vp4 = crc64_nvme_vpclmul_4way_safe(!0, &data) ^ !0; let vp7 = crc64_nvme_vpclmul_7way_safe(!0, &data) ^ !0; + let vp8 = crc64_nvme_vpclmul_8way_safe(!0, &data) ^ !0; assert_eq!(vp2, portable, "2-way mismatch at len={len}"); assert_eq!(vp4, portable, "4-way mismatch at len={len}"); assert_eq!(vp7, portable, "7-way mismatch at len={len}"); + assert_eq!(vp8, portable, "8-way mismatch at len={len}"); } } } diff --git a/src/checksum/diag.rs b/src/checksum/diag.rs index 44c07a81..ede99702 100644 --- a/src/checksum/diag.rs +++ b/src/checksum/diag.rs @@ -38,7 +38,9 @@ pub enum SelectionReason { #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] pub enum Crc32Polynomial { + /// CRC-32/ISO-HDLC, commonly called CRC-32/IEEE. Ieee, + /// CRC-32C using the Castagnoli polynomial. Castagnoli, } @@ -47,50 +49,103 @@ pub enum Crc32Polynomial { #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] pub enum Crc64Polynomial { + /// CRC-64/XZ using the reflected ECMA-182 polynomial. Xz, + /// CRC-64/NVME. Nvme, } +/// Snapshot explaining the CRC-32 kernel selected for one input length. +/// +/// Capability booleans describe facts reported by the active selection policy; +/// use [`crate::platform`] when direct platform capability detection is needed. #[cfg(feature = "crc32")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Crc32SelectionDiag { + /// Polynomial variant evaluated by the selector. pub polynomial: Crc32Polynomial, + /// Input length supplied to the selector, in bytes. pub len: usize, + /// Detected architecture used to choose the dispatch table. pub arch: Arch, + /// High-level reason the reported kernel was selected. pub reason: SelectionReason, + /// Force request after clamping it to detected platform capabilities. pub effective_force: crate::checksum::config::Crc32Force, + /// Name of the policy family that produced this snapshot. pub policy_family: &'static str, + /// Stable name of the selected kernel. pub selected_kernel: &'static str, + /// Stream count reported by the policy. + /// + /// The table-backed policy reports `1`; wider stream selection is encoded in + /// [`Self::selected_kernel`]. pub selected_streams: u8, + /// First size boundary in the active CRC-32 dispatch table. + /// + /// The legacy field name does not guarantee that the next tier uses hardware + /// CRC instructions. pub portable_to_hwcrc: usize, + /// Hardware-CRC-to-fusion boundary reported by the policy. + /// + /// The table-backed policy currently reports its first size boundary here. pub hwcrc_to_fusion: usize, + /// Fusion-to-AVX-512 boundary, or [`usize::MAX`] when not modeled separately. pub fusion_to_avx512: usize, + /// Fusion-to-VPCLMUL boundary, or [`usize::MAX`] when not modeled separately. pub fusion_to_vpclmul: usize, + /// Minimum bytes per reported stream, or [`usize::MAX`] when unavailable. pub min_bytes_per_lane: usize, + /// Whether the policy classified this selection as memory-bound. pub memory_bound: bool, + /// Whether the policy reported a hardware CRC tier. pub has_hwcrc: bool, + /// Whether the policy reported a fused CRC/CLMUL tier. pub has_fusion: bool, + /// Whether the policy reported VPCLMUL support. pub has_vpclmul: bool, + /// Whether the policy reported AVX-512 support. pub has_avx512: bool, + /// Whether the policy reported an AArch64 EOR3 tier. pub has_eor3: bool, + /// Whether the policy reported an AArch64 SVE2 tier. pub has_sve2: bool, } +/// Snapshot explaining the CRC-64 kernel selected for one input length. #[cfg(feature = "crc64")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Crc64SelectionDiag { + /// Polynomial variant evaluated by the selector. pub polynomial: Crc64Polynomial, + /// Input length supplied to the selector, in bytes. pub len: usize, + /// Detected architecture used to choose the dispatch table. pub arch: Arch, + /// High-level reason the reported kernel was selected. pub reason: SelectionReason, + /// Force request after clamping it to detected platform capabilities. pub effective_force: crate::checksum::config::Crc64Force, + /// Name of the policy family that produced this snapshot. pub policy_family: &'static str, + /// Stable name of the selected kernel. pub selected_kernel: &'static str, + /// Stream count reported by the policy. + /// + /// The table-backed policy reports `1`; wider stream selection is encoded in + /// [`Self::selected_kernel`]. pub selected_streams: u8, + /// Boundary between portable and carryless-multiply tiers, in bytes. pub portable_to_clmul: usize, + /// Boundary between narrow and wide carryless-multiply tiers, in bytes. + /// + /// The legacy field name also represents equivalent non-x86 wide tiers. pub pclmul_to_vpclmul: usize, + /// Largest input size assigned to the small-buffer kernel, in bytes. pub small_kernel_max_bytes: usize, + /// Whether the policy selected the four-lane 512-bit VPCLMUL strategy. pub use_4x512: bool, + /// Minimum bytes per reported stream, or [`usize::MAX`] when unavailable. pub min_bytes_per_lane: usize, } diff --git a/src/checksum/dispatchers.rs b/src/checksum/dispatchers.rs index 53e9d9c1..561f77f4 100644 --- a/src/checksum/dispatchers.rs +++ b/src/checksum/dispatchers.rs @@ -17,7 +17,7 @@ /// /// Updated CRC state after processing the input data. #[cfg(feature = "crc16")] -pub type Crc16Fn = fn(u16, &[u8]) -> u16; +pub(in crate::checksum) type Crc16Fn = fn(u16, &[u8]) -> u16; // CRC-24 Kernel Function Type @@ -35,7 +35,7 @@ pub type Crc16Fn = fn(u16, &[u8]) -> u16; /// /// Updated CRC state with the result in the low 24 bits. #[cfg(feature = "crc24")] -pub type Crc24Fn = fn(u32, &[u8]) -> u32; +pub(in crate::checksum) type Crc24Fn = fn(u32, &[u8]) -> u32; // CRC-32 Kernel Function Type @@ -59,7 +59,7 @@ pub type Crc24Fn = fn(u32, &[u8]) -> u32; /// /// Updated CRC state after processing the input data. #[cfg(feature = "crc32")] -pub type Crc32Fn = fn(u32, &[u8]) -> u32; +pub(in crate::checksum) type Crc32Fn = fn(u32, &[u8]) -> u32; // CRC-64 Kernel Function Type @@ -84,4 +84,4 @@ pub type Crc32Fn = fn(u32, &[u8]) -> u32; /// /// Updated CRC state after processing the input data. #[cfg(feature = "crc64")] -pub type Crc64Fn = fn(u64, &[u8]) -> u64; +pub(in crate::checksum) type Crc64Fn = fn(u64, &[u8]) -> u64; diff --git a/src/checksum/kernel_table.rs b/src/checksum/kernel_table.rs index 974679e3..f77317ff 100644 --- a/src/checksum/kernel_table.rs +++ b/src/checksum/kernel_table.rs @@ -331,7 +331,7 @@ pub(crate) struct KernelSet { impl KernelSet { /// Extract the hot-path function pointer set. #[inline] - pub const fn fns(&self) -> KernelFnSet { + const fn fns(&self) -> KernelFnSet { KernelFnSet { #[cfg(feature = "crc16")] crc16_ccitt: self.crc16_ccitt, @@ -352,7 +352,7 @@ impl KernelSet { /// Extract the cold-path name set. #[inline] - pub const fn names(&self) -> KernelNameSet { + const fn names(&self) -> KernelNameSet { KernelNameSet { #[cfg(feature = "crc16")] crc16_ccitt_name: self.crc16_ccitt_name, @@ -410,7 +410,7 @@ const L: usize = 3; impl KernelTable { /// Construct a `KernelTable` from four [`KernelSet`] definitions, splitting /// function pointers from name strings at compile time. - pub const fn from_sets( + const fn from_sets( requires: Caps, boundaries: [usize; 3], xs: KernelSet, @@ -428,7 +428,7 @@ impl KernelTable { /// Select the hot-path function pointer set for the given buffer length. #[inline] - pub const fn select_fns(&self, len: usize) -> &KernelFnSet { + pub(in crate::checksum) const fn select_fns(&self, len: usize) -> &KernelFnSet { if len <= self.boundaries[0] { &self.fns[XS] } else if len <= self.boundaries[1] { @@ -444,7 +444,7 @@ impl KernelTable { /// /// Used only for introspection / diagnostics; never called on the hot path. #[inline] - pub const fn select_names(&self, len: usize) -> &KernelNameSet { + pub(in crate::checksum) const fn select_names(&self, len: usize) -> &KernelNameSet { if len <= self.boundaries[0] { &self.names[XS] } else if len <= self.boundaries[1] { @@ -460,10 +460,9 @@ impl KernelTable { /// /// A table is hardware-accelerated if it requires any CPU capabilities /// beyond the baseline (i.e., `requires != Caps::NONE`). - #[cfg(any(test, feature = "diag"))] - #[cfg_attr(test, allow(dead_code))] + #[cfg(feature = "diag")] #[inline] - pub const fn is_hardware_accelerated(&self) -> bool { + const fn is_hardware_accelerated(&self) -> bool { !self.requires.is_empty() } } @@ -474,6 +473,13 @@ impl KernelTable { /// Accepts the same field layout as the old monolithic `KernelTable { ... }` /// struct literal (with `requires`, `boundaries`, `xs`, `s`, `m`, `l` labels), /// converting it to `KernelTable::from_sets(...)` under the hood. +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" +))] macro_rules! kernel_table { ( requires: $req:expr, @@ -488,8 +494,7 @@ macro_rules! kernel_table { } /// Returns `true` if the active kernel table uses hardware-accelerated CRC kernels. -#[cfg(any(test, feature = "diag"))] -#[cfg_attr(test, allow(dead_code))] +#[cfg(feature = "diag")] #[inline] pub fn is_hardware_accelerated() -> bool { active_table().is_hardware_accelerated() @@ -669,18 +674,6 @@ fn capability_match(caps: Caps) -> Option<&'static KernelTable> { #[cfg(feature = "crc64")] #[inline] fn capability_match_crc64(caps: Caps) -> Option<&'static KernelTable> { - #[cfg(target_arch = "riscv64")] - { - use crate::platform::caps::riscv::{V, ZBC, ZVBC}; - let v_zvbc = V.union(ZVBC); - if caps.has(v_zvbc) { - return Some(&RISCV64_CRC64_ZVBC_TABLE); - } - if caps.has(ZBC) { - return Some(&RISCV64_CRC64_ZBC_TABLE); - } - } - capability_match(caps) } @@ -720,7 +713,7 @@ const PORTABLE_SET: KernelSet = KernelSet { crc64_nvme_name: "portable/slice16", }; -pub static PORTABLE_TABLE: KernelTable = KernelTable::from_sets( +static PORTABLE_TABLE: KernelTable = KernelTable::from_sets( Caps::NONE, [64, 256, 4096], PORTABLE_SET, @@ -743,14 +736,24 @@ mod aarch64_tables { #[cfg(feature = "crc64")] use crate::checksum::crc64::kernels::aarch64 as crc64_k; - #[cfg(all(feature = "crc16", not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all( + feature = "crc16", + feature = "std", + not(miri), + any(target_os = "linux", target_os = "android") + ))] const G3_CRC16_PMULL_EOR3_2WAY_MAX_LEN: usize = 262_144; /// Graviton3 CRC16/CCITT large-path PMULL+EOR3 hybrid. /// /// PMULL+EOR3 2-way improves lower "large" buffers, while PMULL+EOR3 1-way /// remains safer for very large buffers on G3. - #[cfg(all(feature = "crc16", not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all( + feature = "crc16", + feature = "std", + not(miri), + any(target_os = "linux", target_os = "android") + ))] #[inline] fn g3_crc16_ccitt_l_hybrid(crc: u16, data: &[u8]) -> u16 { if data.len() <= G3_CRC16_PMULL_EOR3_2WAY_MAX_LEN { @@ -764,7 +767,12 @@ mod aarch64_tables { /// /// Empirically, 2-way PMULL+EOR3 helps lower "large" buffers while 1-way /// PMULL+EOR3 remains safer for very large buffers on G3. - #[cfg(all(feature = "crc16", not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all( + feature = "crc16", + feature = "std", + not(miri), + any(target_os = "linux", target_os = "android") + ))] #[inline] fn g3_crc16_ibm_l_hybrid(crc: u16, data: &[u8]) -> u16 { if data.len() <= G3_CRC16_PMULL_EOR3_2WAY_MAX_LEN { @@ -777,7 +785,7 @@ mod aarch64_tables { // Apple M1-M3 table. // // Features: PMULL + SHA3 (EOR3) - pub static APPLE_M1M3_TABLE: KernelTable = kernel_table! { + pub(super) static APPLE_M1M3_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::aarch64::CRC_READY .union(crate::platform::caps::aarch64::PMULL_EOR3_READY) .union(crate::platform::caps::aarch64::PMULL_READY), @@ -911,8 +919,8 @@ mod aarch64_tables { // Graviton2 table. // // Features: PMULL (no EOR3/SHA3) - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] - pub static GRAVITON2_TABLE: KernelTable = kernel_table! { + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] + pub(super) static GRAVITON2_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::aarch64::CRC_READY.union(crate::platform::caps::aarch64::PMULL_EOR3_READY), boundaries: [64, 256, 4096], @@ -1044,7 +1052,7 @@ mod aarch64_tables { // Graviton3 table. // // Features: PMULL + SHA3/EOR3 - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] const G3_XS: KernelSet = KernelSet { #[cfg(feature = "crc16")] crc16_ccitt: crc16_k::CCITT_PMULL_SMALL_KERNEL, @@ -1076,7 +1084,7 @@ mod aarch64_tables { crc64_nvme_name: "aarch64/pmull-small", }; - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] const G3_S: KernelSet = KernelSet { #[cfg(feature = "crc16")] crc16_ccitt: crc16_k::CCITT_PMULL[0], @@ -1108,7 +1116,7 @@ mod aarch64_tables { crc64_nvme_name: "aarch64/pmull", }; - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] const G3_M: KernelSet = KernelSet { #[cfg(feature = "crc16")] crc16_ccitt: crc16_k::CCITT_PMULL_EOR3[0], // PMULL+EOR3 cuts XOR chain in large lanes. @@ -1140,7 +1148,7 @@ mod aarch64_tables { crc64_nvme_name: "aarch64/pmull-2way", }; - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] const G3_L: KernelSet = KernelSet { #[cfg(feature = "crc16")] crc16_ccitt: g3_crc16_ccitt_l_hybrid, @@ -1172,8 +1180,8 @@ mod aarch64_tables { crc64_nvme_name: "aarch64/pmull-eor3", }; - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] - pub static GRAVITON3_TABLE: KernelTable = KernelTable::from_sets( + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] + pub(super) static GRAVITON3_TABLE: KernelTable = KernelTable::from_sets( crate::platform::caps::aarch64::CRC_READY.union(crate::platform::caps::aarch64::PMULL_EOR3_READY), [64, 256, 4096], G3_XS, @@ -1185,8 +1193,8 @@ mod aarch64_tables { // Graviton4 Table // // Starts from Graviton3, but keeps 2-way PMULL+EOR3 for CRC16 large classes. - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] - pub static GRAVITON4_TABLE: KernelTable = KernelTable::from_sets( + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] + pub(super) static GRAVITON4_TABLE: KernelTable = KernelTable::from_sets( crate::platform::caps::aarch64::CRC_READY.union(crate::platform::caps::aarch64::PMULL_EOR3_READY), [64, 256, 4096], G3_XS, @@ -1200,7 +1208,26 @@ mod aarch64_tables { crc16_ibm: crc16_k::IBM_PMULL_EOR3[1], #[cfg(feature = "crc16")] crc16_ibm_name: "aarch64/pmull-eor3-2way", - ..G3_M + #[cfg(feature = "crc24")] + crc24_openpgp: crc24_k::OPENPGP_PMULL[0], + #[cfg(feature = "crc24")] + crc24_openpgp_name: "aarch64/pmull", + #[cfg(feature = "crc32")] + crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, + #[cfg(feature = "crc32")] + crc32_ieee_name: "aarch64/pmull-small", + #[cfg(feature = "crc32")] + crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, + #[cfg(feature = "crc32")] + crc32c_name: "aarch64/pmull-small", + #[cfg(feature = "crc64")] + crc64_xz: crc64_k::XZ_PMULL[0], + #[cfg(feature = "crc64")] + crc64_xz_name: "aarch64/pmull", + #[cfg(feature = "crc64")] + crc64_nvme: crc64_k::NVME_PMULL[1], + #[cfg(feature = "crc64")] + crc64_nvme_name: "aarch64/pmull-2way", }, KernelSet { #[cfg(feature = "crc16")] @@ -1211,7 +1238,26 @@ mod aarch64_tables { crc16_ibm: crc16_k::IBM_PMULL_EOR3[1], #[cfg(feature = "crc16")] crc16_ibm_name: "aarch64/pmull-eor3-2way", - ..G3_L + #[cfg(feature = "crc24")] + crc24_openpgp: crc24_k::OPENPGP_PMULL[0], + #[cfg(feature = "crc24")] + crc24_openpgp_name: "aarch64/pmull", + #[cfg(feature = "crc32")] + crc32_ieee: crc32_k::CRC32_PMULL_EOR3[0], + #[cfg(feature = "crc32")] + crc32_ieee_name: "aarch64/pmull-eor3-v9s3x2e-s3", + #[cfg(feature = "crc32")] + crc32c: crc32_k::CRC32C_PMULL_EOR3[0], + #[cfg(feature = "crc32")] + crc32c_name: "aarch64/pmull-eor3-v9s3x2e-s3", + #[cfg(feature = "crc64")] + crc64_xz: crc64_k::XZ_PMULL_EOR3[0], + #[cfg(feature = "crc64")] + crc64_xz_name: "aarch64/pmull-eor3", + #[cfg(feature = "crc64")] + crc64_nvme: crc64_k::NVME_PMULL_EOR3[0], + #[cfg(feature = "crc64")] + crc64_nvme_name: "aarch64/pmull-eor3", }, ); @@ -1219,12 +1265,12 @@ mod aarch64_tables { // // For unknown ARM platforms with PMULL + SHA3 features. // Uses Apple M1-M3 selections (good EOR3 support). - pub static GENERIC_ARM_PMULL_EOR3_TABLE: KernelTable = APPLE_M1M3_TABLE; + pub(super) static GENERIC_ARM_PMULL_EOR3_TABLE: KernelTable = APPLE_M1M3_TABLE; // Generic ARM PMULL Table (conservative) // // For unknown ARM platforms with CRC + PMULL but *without* SHA3/EOR3. - pub static GENERIC_ARM_PMULL_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_ARM_PMULL_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::aarch64::CRC_READY.union(crate::platform::caps::aarch64::PMULL_READY), boundaries: [64, 256, 4096], @@ -1354,7 +1400,7 @@ mod aarch64_tables { }; /// PMULL-only table for platforms without the CRC extension. - pub static GENERIC_ARM_PMULL_NO_CRC_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_ARM_PMULL_NO_CRC_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::aarch64::PMULL_READY, boundaries: [64, 256, 4096], @@ -1484,7 +1530,7 @@ mod aarch64_tables { }; /// CRC-only table for platforms without PMULL. - pub static GENERIC_ARM_CRC_ONLY_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_ARM_CRC_ONLY_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::aarch64::CRC_READY, boundaries: [64, 256, 4096], @@ -1648,7 +1694,7 @@ mod x86_64_tables { // Zen4 table. // // Features: VPCLMULQDQ + AVX-512 - pub static ZEN4_TABLE: KernelTable = kernel_table! { + pub(super) static ZEN4_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::x86::VPCLMUL_READY .union(crate::platform::caps::x86::PCLMUL_READY) .union(crate::platform::caps::x86::CRC32C_READY), @@ -1783,12 +1829,12 @@ mod x86_64_tables { // // For unknown x86-64 platforms with VPCLMULQDQ. // Uses Zen4 selections (good AVX-512/VPCLMUL support). - pub static GENERIC_X86_VPCLMUL_TABLE: KernelTable = ZEN4_TABLE; + pub(super) static GENERIC_X86_VPCLMUL_TABLE: KernelTable = ZEN4_TABLE; /// VPCLMUL table that never selects SSE4.2 CRC32C instructions/fusion. /// /// Use on systems with VPCLMUL but without SSE4.2 (`CRC32C_READY`). - pub static GENERIC_X86_VPCLMUL_NO_CRC32C_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_X86_VPCLMUL_NO_CRC32C_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::x86::VPCLMUL_READY.union(crate::platform::caps::x86::PCLMUL_READY), boundaries: [64, 256, 4096], @@ -1920,7 +1966,7 @@ mod x86_64_tables { // Generic x86-64 PCLMUL Table (conservative) // // Features: PCLMULQDQ only - pub static GENERIC_X86_PCLMUL_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_X86_PCLMUL_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::x86::PCLMUL_READY.union(crate::platform::caps::x86::CRC32C_READY), boundaries: [64, 256, 4096], @@ -2052,7 +2098,7 @@ mod x86_64_tables { /// PCLMUL table that never selects SSE4.2 CRC32C instructions/fusion. /// /// Use on systems with PCLMUL but without SSE4.2 (`CRC32C_READY`). - pub static GENERIC_X86_PCLMUL_NO_CRC32C_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_X86_PCLMUL_NO_CRC32C_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::x86::PCLMUL_READY, boundaries: [64, 256, 4096], @@ -2182,7 +2228,7 @@ mod x86_64_tables { }; /// SSE4.2-only table: accelerate CRC32C, keep other variants portable. - pub static GENERIC_X86_CRC32C_ONLY_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_X86_CRC32C_ONLY_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::x86::CRC32C_READY, boundaries: [64, 256, 4096], @@ -2313,7 +2359,7 @@ mod x86_64_tables { } #[cfg(target_arch = "x86_64")] -pub use x86_64_tables::*; +use x86_64_tables::*; // s390x Platform Tables @@ -2329,7 +2375,7 @@ mod s390x_tables { #[cfg(feature = "crc64")] use crate::checksum::crc64::kernels::s390x as crc64_k; - pub static S390X_Z13_TABLE: KernelTable = kernel_table! { + pub(super) static S390X_Z13_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::s390x::Z13_READY, boundaries: [64, 256, 4096], xs: PORTABLE_SET, @@ -2396,7 +2442,7 @@ mod s390x_tables { }, }; - pub static S390X_Z14_TABLE: KernelTable = kernel_table! { + pub(super) static S390X_Z14_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::s390x::Z13_READY, boundaries: [64, 128, 4096], xs: PORTABLE_SET, @@ -2463,7 +2509,7 @@ mod s390x_tables { }, }; - pub static S390X_Z15_TABLE: KernelTable = kernel_table! { + pub(super) static S390X_Z15_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::s390x::Z13_READY, boundaries: [63, 63, 4096], xs: PORTABLE_SET, @@ -2532,7 +2578,7 @@ mod s390x_tables { } #[cfg(target_arch = "s390x")] -pub use s390x_tables::*; +use s390x_tables::*; // powerpc64 Platform Tables @@ -2548,7 +2594,7 @@ mod power_tables { #[cfg(feature = "crc64")] use crate::checksum::crc64::kernels::power as crc64_k; - pub static POWER8_TABLE: KernelTable = kernel_table! { + pub(super) static POWER8_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::power::VPMSUM_READY, boundaries: [64, 128, 4096], xs: PORTABLE_SET, @@ -2615,7 +2661,7 @@ mod power_tables { }, }; - pub static POWER9_TABLE: KernelTable = kernel_table! { + pub(super) static POWER9_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::power::VPMSUM_READY, boundaries: [64, 64, 4096], xs: PORTABLE_SET, @@ -2682,7 +2728,7 @@ mod power_tables { }, }; - pub static POWER10_TABLE: KernelTable = kernel_table! { + pub(super) static POWER10_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::power::VPMSUM_READY, boundaries: [63, 63, 4096], xs: PORTABLE_SET, @@ -2751,7 +2797,7 @@ mod power_tables { } #[cfg(target_arch = "powerpc64")] -pub use power_tables::*; +use power_tables::*; // riscv64 Platform Tables @@ -2764,10 +2810,8 @@ mod riscv64_tables { use crate::checksum::crc24::kernels::riscv64 as crc24_k; #[cfg(feature = "crc32")] use crate::checksum::crc32::kernels::riscv64 as crc32_k; - #[cfg(feature = "crc64")] - use crate::checksum::crc64::kernels::riscv64 as crc64_k; - pub static RISCV64_ZBC_TABLE: KernelTable = kernel_table! { + pub(super) static RISCV64_ZBC_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::riscv::ZBC, boundaries: [63, 1024, 4096], xs: PORTABLE_SET, @@ -2866,7 +2910,7 @@ mod riscv64_tables { }, }; - pub static RISCV64_ZVBC_TABLE: KernelTable = kernel_table! { + pub(super) static RISCV64_ZVBC_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::riscv::V.union(crate::platform::caps::riscv::ZVBC), boundaries: [63, 1024, 4096], xs: PORTABLE_SET, @@ -2892,13 +2936,13 @@ mod riscv64_tables { #[cfg(feature = "crc32")] crc32c_name: "riscv64/zvbc", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_ZVBC[0], + crc64_xz: crate::checksum::crc64::portable::crc64_slice16_xz, #[cfg(feature = "crc64")] - crc64_xz_name: "riscv64/zvbc", + crc64_xz_name: "portable/slice16", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_ZVBC[0], + crc64_nvme: crate::checksum::crc64::portable::crc64_slice16_nvme, #[cfg(feature = "crc64")] - crc64_nvme_name: "riscv64/zvbc", + crc64_nvme_name: "portable/slice16", }, m: KernelSet { #[cfg(feature = "crc16")] @@ -2922,13 +2966,13 @@ mod riscv64_tables { #[cfg(feature = "crc32")] crc32c_name: "riscv64/zvbc-2way", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_ZVBC[1], + crc64_xz: crate::checksum::crc64::portable::crc64_slice16_xz, #[cfg(feature = "crc64")] - crc64_xz_name: "riscv64/zvbc-2way", + crc64_xz_name: "portable/slice16", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_ZVBC[1], + crc64_nvme: crate::checksum::crc64::portable::crc64_slice16_nvme, #[cfg(feature = "crc64")] - crc64_nvme_name: "riscv64/zvbc-2way", + crc64_nvme_name: "portable/slice16", }, l: KernelSet { #[cfg(feature = "crc16")] @@ -2952,44 +2996,19 @@ mod riscv64_tables { #[cfg(feature = "crc32")] crc32c_name: "riscv64/zvbc-4way", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_ZVBC[2], + crc64_xz: crate::checksum::crc64::portable::crc64_slice16_xz, #[cfg(feature = "crc64")] - crc64_xz_name: "riscv64/zvbc-4way", + crc64_xz_name: "portable/slice16", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_ZVBC[2], + crc64_nvme: crate::checksum::crc64::portable::crc64_slice16_nvme, #[cfg(feature = "crc64")] - crc64_nvme_name: "riscv64/zvbc-4way", + crc64_nvme_name: "portable/slice16", }, }; - - #[cfg(feature = "crc64")] - // Keep CRC64 auto on portable slice-by-16 for now. - // - // No tracked target evidence currently justifies selecting the accelerated - // CRC64 kernels by default. - #[cfg(feature = "crc64")] - pub static RISCV64_CRC64_ZBC_TABLE: KernelTable = kernel_table! { - requires: crate::platform::caps::riscv::ZBC, - boundaries: [128, 4096, 16384], - xs: PORTABLE_SET, - s: PORTABLE_SET, - m: PORTABLE_SET, - l: PORTABLE_SET, - }; - - #[cfg(feature = "crc64")] - pub static RISCV64_CRC64_ZVBC_TABLE: KernelTable = kernel_table! { - requires: crate::platform::caps::riscv::V.union(crate::platform::caps::riscv::ZVBC), - boundaries: [63, 1024, 4096], - xs: PORTABLE_SET, - s: PORTABLE_SET, - m: PORTABLE_SET, - l: PORTABLE_SET, - }; } #[cfg(target_arch = "riscv64")] -pub use riscv64_tables::*; +use riscv64_tables::*; // Tests @@ -3004,14 +3023,55 @@ mod tests { // Verify portable table returns correct sets for each size class let table = &PORTABLE_TABLE; - assert!(core::ptr::eq(table.select_fns(0), &table.fns[XS])); - assert!(core::ptr::eq(table.select_fns(64), &table.fns[XS])); - assert!(core::ptr::eq(table.select_fns(65), &table.fns[S])); - assert!(core::ptr::eq(table.select_fns(256), &table.fns[S])); - assert!(core::ptr::eq(table.select_fns(257), &table.fns[M])); - assert!(core::ptr::eq(table.select_fns(4096), &table.fns[M])); - assert!(core::ptr::eq(table.select_fns(4097), &table.fns[L])); - assert!(core::ptr::eq(table.select_fns(1_000_000), &table.fns[L])); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(0)), + core::ptr::from_ref(&table.fns[XS]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(64)), + core::ptr::from_ref(&table.fns[XS]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(65)), + core::ptr::from_ref(&table.fns[S]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(256)), + core::ptr::from_ref(&table.fns[S]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(257)), + core::ptr::from_ref(&table.fns[M]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(4096)), + core::ptr::from_ref(&table.fns[M]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(4097)), + core::ptr::from_ref(&table.fns[L]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(1_000_000)), + core::ptr::from_ref(&table.fns[L]) + )); + } + + #[test] + #[cfg(all(target_arch = "x86_64", feature = "crc64"))] + fn test_vpclmul_without_crc32c_selects_crc64_xz_4x512_above_4096() { + use crate::platform::caps::x86::{CRC32C_READY, VPCLMUL_READY}; + + assert!(!VPCLMUL_READY.has(CRC32C_READY)); + let table = select_crc64_table(VPCLMUL_READY); + assert!(core::ptr::eq(table, &GENERIC_X86_VPCLMUL_NO_CRC32C_TABLE)); + assert_eq!(table.select_names(4096).crc64_xz_name, "x86_64/vpclmul-2way"); + assert_eq!(table.select_names(4097).crc64_xz_name, "x86_64/vpclmul-4x512"); + assert_eq!(table.select_names(4097).crc64_nvme_name, "x86_64/vpclmul-4way"); + assert!(core::ptr::fn_addr_eq( + table.select_fns(4097).crc64_xz, + crate::checksum::crc64::kernels::x86_64::XZ_VPCLMUL_4X512 + )); } #[test] @@ -3045,7 +3105,7 @@ mod tests { #[cfg(feature = "crc64")] { - let crc64_names = RISCV64_CRC64_ZVBC_TABLE.select_names(64); + let crc64_names = RISCV64_ZVBC_TABLE.select_names(64); assert_eq!(crc64_names.crc64_xz_name, "portable/slice16"); assert_eq!(crc64_names.crc64_nvme_name, "portable/slice16"); } @@ -3055,7 +3115,10 @@ mod tests { fn test_select_table_fallback() { // With no capabilities, should return portable table let table = select_table(Caps::NONE); - assert!(core::ptr::eq(table, &PORTABLE_TABLE)); + assert!(core::ptr::eq( + core::ptr::from_ref(table), + core::ptr::from_ref(&PORTABLE_TABLE) + )); } // Oneshot Function Tests @@ -3146,7 +3209,7 @@ mod tests { let sizes = [1, 64, 65, 256, 257, 4096, 4097, 65536]; for &size in &sizes { - let data: Vec = (0..size).map(|i| (i % 256) as u8).collect(); + let data: Vec = (0u8..=u8::MAX).cycle().take(size).collect(); // Just verify no panics and consistent non-zero results for non-empty data #[cfg(feature = "crc64")] let _ = crc64_xz(&data); diff --git a/src/checksum/macros.rs b/src/checksum/macros.rs index 7737a4f7..509fcfcc 100644 --- a/src/checksum/macros.rs +++ b/src/checksum/macros.rs @@ -42,10 +42,7 @@ macro_rules! crc_vectored_dispatch { /// This covers the repeated "resolve once under std, otherwise use auto" /// pattern shared by CRC-16/32/64 variants while leaving the actual kernel /// wrappers and force-specific architecture helpers local to each module. -#[cfg_attr( - not(any(feature = "crc16", feature = "crc32", feature = "crc64")), - allow(unused_macros) -)] +#[cfg(any(feature = "crc16", feature = "crc32", feature = "crc64"))] macro_rules! define_crc_dispatch { ( word_ty: $word_ty:ty, @@ -190,7 +187,6 @@ macro_rules! define_buffered_crc { /// /// Data is buffered internally until enough accumulates for efficient /// SIMD processing. - #[allow(clippy::indexing_slicing)] // Safety: All slice indices are bounds-checked by the algorithm: // - self.len < buffer_size (invariant maintained by this function) // - fill = min(input.len(), space), so input[..fill] and buffer[len..len+fill] are valid @@ -250,7 +246,6 @@ macro_rules! define_buffered_crc { /// /// Flushes any remaining buffered data before computing the final CRC. #[must_use] - #[allow(clippy::indexing_slicing)] // Safety: self.len < buffer_size (invariant) pub fn finalize(&self) -> <$inner as $crate::Checksum>::Output { if self.len > 0 { diff --git a/src/checksum/mod.rs b/src/checksum/mod.rs index b0f1c2c3..a32de765 100644 --- a/src/checksum/mod.rs +++ b/src/checksum/mod.rs @@ -47,10 +47,10 @@ //! ```toml //! [dependencies] //! # Smallest CRC-32-only build -//! rscrypto = { version = "0.7", default-features = false, features = ["crc32"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["crc32"] } //! //! # All checksum families -//! rscrypto = { version = "0.7", default-features = false, features = ["checksums"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["checksums"] } //! ``` //! //! # API Conventions @@ -76,7 +76,7 @@ //! //! ```toml //! [dependencies] -//! rscrypto = { version = "0.7", default-features = false, features = ["crc32"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["crc32"] } //! ``` mod common; @@ -104,6 +104,7 @@ pub mod io; #[doc(hidden)] pub(crate) mod kernel_table; #[cfg(feature = "alloc")] +/// Buffered checksum adapters for workloads composed of many short updates. pub mod buffered { #[cfg(feature = "crc16")] pub use crate::checksum::crc16::{BufferedCrc16Ccitt, BufferedCrc16Ibm}; diff --git a/src/hashes/common/mod.rs b/src/hashes/common/mod.rs index 2875ebe6..3b8dd548 100644 --- a/src/hashes/common/mod.rs +++ b/src/hashes/common/mod.rs @@ -3,4 +3,4 @@ //! This module currently owns the AArch64 BLAKE3 prefetch helper. #[cfg(all(feature = "blake3", target_arch = "aarch64"))] -pub mod prefetch; +pub(crate) mod prefetch; diff --git a/src/hashes/common/prefetch.rs b/src/hashes/common/prefetch.rs index 83952c65..1887684f 100644 --- a/src/hashes/common/prefetch.rs +++ b/src/hashes/common/prefetch.rs @@ -8,7 +8,6 @@ // SAFETY: This module provides low-level prefetch intrinsics that require unsafe. // Prefetch instructions are hints to the CPU and cannot cause memory unsafety; // invalid addresses are silently ignored. -#![allow(unsafe_code)] /// Prefetch data for read into L1 cache (PLDL1KEEP). /// diff --git a/src/hashes/crypto/ascon.rs b/src/hashes/crypto/ascon.rs index a6032128..ab3e7810 100644 --- a/src/hashes/crypto/ascon.rs +++ b/src/hashes/crypto/ascon.rs @@ -2,8 +2,6 @@ //! //! Portable, `no_std`, pure Rust implementation. -#![allow(clippy::indexing_slicing)] // Fixed-size state + sponge buffering - use core::fmt; use crate::{ @@ -11,18 +9,21 @@ use crate::{ traits::{Digest, Xof}, }; -#[cfg(target_arch = "aarch64")] +#[cfg(all(target_arch = "aarch64", any(test, feature = "std")))] mod aarch64; #[doc(hidden)] +#[cfg(any(test, feature = "std"))] pub(crate) mod dispatch; #[doc(hidden)] +#[cfg(any(test, feature = "std"))] pub(crate) mod dispatch_tables; #[cfg(test)] mod kernel_test; +#[cfg(any(test, feature = "std"))] pub(crate) mod kernels; -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", any(test, feature = "std")))] mod x86_64_avx2; -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", any(test, feature = "std")))] mod x86_64_avx512; const RATE: usize = 8; @@ -48,7 +49,7 @@ impl Permuter for InlinePermuter { // Ascon permutation round constants (12 rounds). // Used by SIMD kernels; the shared portable permutation inlines the constants. -#[allow(dead_code)] +#[cfg(all(any(test, feature = "std"), any(target_arch = "aarch64", target_arch = "x86_64")))] const RC: [u64; 12] = [0xF0, 0xE1, 0xD2, 0xC3, 0xB4, 0xA5, 0x96, 0x87, 0x78, 0x69, 0x5A, 0x4B]; // Domain-specific IVs (from the Ascon hash/XOF specification). @@ -80,7 +81,7 @@ define_unit_error! { const fn pad(n: usize) -> u64 { // Produce the padding mask used by the reference construction: // XOR `pad(len)` into state[0], with state interpreted little-endian. - 0x01_u64 << (8 * n) + 0x01_u64 << n.strict_mul(8) } #[derive(Clone)] @@ -121,7 +122,7 @@ impl( ) { debug_assert_eq!(inputs.len(), N); let len = inputs[0].len(); - let full_bytes = len / RATE * RATE; + let full_bytes = len.strict_sub(len % RATE); for off in (0..full_bytes).step_by(RATE) { for (state, input) in states.iter_mut().zip(inputs.iter().copied()) { let mut block = [0u8; RATE]; - block.copy_from_slice(&input[off..off + RATE]); + block.copy_from_slice(&input[off..off.strict_add(RATE)]); state[0] ^= u64::from_le_bytes(block); } permute_many(states); } - let tail_len = len - full_bytes; + let tail_len = len.strict_sub(full_bytes); for (state, input) in states.iter_mut().zip(inputs.iter().copied()) { let mut block = [0u8; RATE]; block[..tail_len].copy_from_slice(&input[full_bytes..]); @@ -259,10 +260,10 @@ fn squeeze_hash256_group( let mut off = 0usize; while off < 24 { for (state, output) in states.iter().zip(outputs.iter_mut()) { - output[off..off + RATE].copy_from_slice(&state[0].to_le_bytes()); + output[off..off.strict_add(RATE)].copy_from_slice(&state[0].to_le_bytes()); } permute_many(states); - off += RATE; + off = off.strict_add(RATE); } for (state, output) in states.iter().zip(outputs.iter_mut()) { output[24..32].copy_from_slice(&state[0].to_le_bytes()); @@ -276,14 +277,14 @@ fn squeeze_xof_group( outputs: &mut [u8], permute_many: fn(&mut [[u64; 5]; N]), ) { - debug_assert_eq!(outputs.len(), N * out_len); + debug_assert_eq!(outputs.len(), N.strict_mul(out_len)); let mut produced = 0usize; while produced < out_len { - let take = core::cmp::min(RATE, out_len - produced); + let take = core::cmp::min(RATE, out_len.strict_sub(produced)); for (state, output) in states.iter().zip(outputs.chunks_exact_mut(out_len)) { - output[produced..produced + take].copy_from_slice(&state[0].to_le_bytes()[..take]); + output[produced..produced.strict_add(take)].copy_from_slice(&state[0].to_le_bytes()[..take]); } - produced += take; + produced = produced.strict_add(take); if produced < out_len { permute_many(states); } @@ -321,7 +322,7 @@ fn xof_many_equal_len_group( permute_many: fn(&mut [[u64; 5]; N]), ) { debug_assert_eq!(inputs.len(), N); - debug_assert_eq!(outputs.len(), N * out_len); + debug_assert_eq!(outputs.len(), N.strict_mul(out_len)); let mut states = init_states::(iv); absorb_equal_len_group(&mut states, inputs, permute_many); squeeze_xof_group(&mut states, out_len, outputs, permute_many); @@ -383,17 +384,19 @@ impl AsconHash256 { match kid { kernels::AsconPermute12KernelId::Portable => { - let mut input_groups = inputs.chunks_exact(1); - let mut output_groups = outputs.chunks_exact_mut(1); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<1>(); + let (output_groups, rem_outputs) = outputs.as_chunks_mut::<1>(); + debug_assert!(rem_inputs.is_empty()); + debug_assert!(rem_outputs.is_empty()); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.iter_mut()) { digest_many_equal_len_group::<1>(group_inputs, group_outputs, HASH256_IV, permute_12_many_portable::<1>); } } #[cfg(target_arch = "aarch64")] kernels::AsconPermute12KernelId::Aarch64Neon => { - let mut input_groups = inputs.chunks_exact(2); - let mut output_groups = outputs.chunks_exact_mut(2); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<2>(); + let (output_groups, rem_outputs) = outputs.as_chunks_mut::<2>(); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.iter_mut()) { digest_many_equal_len_group::<2>( group_inputs, group_outputs, @@ -401,8 +404,6 @@ impl AsconHash256 { aarch64::permute_12_aarch64_neon_x2, ); } - let rem_inputs = input_groups.remainder(); - let rem_outputs = output_groups.into_remainder(); if !rem_inputs.is_empty() { Self::digest_many_with_kernel( dispatch::batch_fallback_kernel_id(kid, rem_inputs.len()), @@ -413,9 +414,9 @@ impl AsconHash256 { } #[cfg(target_arch = "x86_64")] kernels::AsconPermute12KernelId::X86Avx2 => { - let mut input_groups = inputs.chunks_exact(4); - let mut output_groups = outputs.chunks_exact_mut(4); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<4>(); + let (output_groups, rem_outputs) = outputs.as_chunks_mut::<4>(); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.iter_mut()) { digest_many_equal_len_group::<4>( group_inputs, group_outputs, @@ -423,8 +424,6 @@ impl AsconHash256 { x86_64_avx2::permute_12_x86_avx2_x4, ); } - let rem_inputs = input_groups.remainder(); - let rem_outputs = output_groups.into_remainder(); if !rem_inputs.is_empty() { Self::digest_many_with_kernel( dispatch::batch_fallback_kernel_id(kid, rem_inputs.len()), @@ -435,9 +434,9 @@ impl AsconHash256 { } #[cfg(target_arch = "x86_64")] kernels::AsconPermute12KernelId::X86Avx512 => { - let mut input_groups = inputs.chunks_exact(8); - let mut output_groups = outputs.chunks_exact_mut(8); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<8>(); + let (output_groups, rem_outputs) = outputs.as_chunks_mut::<8>(); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.iter_mut()) { digest_many_equal_len_group::<8>( group_inputs, group_outputs, @@ -445,8 +444,6 @@ impl AsconHash256 { x86_64_avx512::permute_12_x86_avx512_x8, ); } - let rem_inputs = input_groups.remainder(); - let rem_outputs = output_groups.into_remainder(); if !rem_inputs.is_empty() { Self::digest_many_with_kernel( dispatch::batch_fallback_kernel_id(kid, rem_inputs.len()), @@ -472,12 +469,12 @@ impl AsconHash256 { let mut start = 0usize; while start < inputs.len() { let len = inputs[start].len(); - let mut end = start + 1; + let mut end = start.strict_add(1); while end < inputs.len() && inputs[end].len() == len { - end += 1; + end = end.strict_add(1); } - let kid = Self::batch_kernel_id_for_count(end - start); + let kid = Self::batch_kernel_id_for_count(end.strict_sub(start)); Self::digest_many_with_kernel(kid, &inputs[start..end], &mut outputs[start..end]); start = end; } @@ -531,12 +528,14 @@ impl fmt::Debug for AsconXof { } impl AsconXof { + /// Creates an empty Ascon-XOF128 state. #[inline] #[must_use] pub fn new() -> Self { Self::default() } + /// Absorbs `data` and returns an Ascon-XOF128 output reader. #[inline] #[must_use] pub fn xof(data: &[u8]) -> AsconXofReader { @@ -544,16 +543,19 @@ impl AsconXof { AsconXofReader::from_state(state) } + /// Absorbs more input into the XOF state. #[inline] pub fn update(&mut self, data: &[u8]) { self.sponge.update(data); } + /// Resets the XOF state to its initial value. #[inline] pub fn reset(&mut self) { *self = Self::default(); } + /// Finalizes the current state into an output reader without consuming it. #[inline] #[must_use] pub fn finalize_xof(&self) -> AsconXofReader { @@ -592,10 +594,10 @@ impl AsconXof { ) { assert_eq!( outputs.len(), - inputs.len() * out_len, + inputs.len().strict_mul(out_len), "input/output batch length mismatch" ); - if inputs.is_empty() { + if inputs.is_empty() || out_len == 0 { return; } @@ -604,17 +606,18 @@ impl AsconXof { if degree == 1 || !inputs_have_equal_len(inputs) { let scalar_kid = dispatch::scalar_kernel_id(); for (index, input) in inputs.iter().enumerate() { - let base = index * out_len; - Self::hash_into_with_kernel(scalar_kid, input, &mut outputs[base..base + out_len]); + let base = index.strict_mul(out_len); + Self::hash_into_with_kernel(scalar_kid, input, &mut outputs[base..base.strict_add(out_len)]); } return; } match kid { kernels::AsconPermute12KernelId::Portable => { - let mut input_groups = inputs.chunks_exact(1); + let (input_groups, rem_inputs) = inputs.as_chunks::<1>(); let mut output_groups = outputs.chunks_exact_mut(out_len); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + debug_assert!(rem_inputs.is_empty()); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.by_ref()) { xof_many_equal_len_group::<1>( group_inputs, out_len, @@ -626,9 +629,9 @@ impl AsconXof { } #[cfg(target_arch = "aarch64")] kernels::AsconPermute12KernelId::Aarch64Neon => { - let mut input_groups = inputs.chunks_exact(2); - let mut output_groups = outputs.chunks_exact_mut(2 * out_len); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<2>(); + let mut output_groups = outputs.chunks_exact_mut(2usize.strict_mul(out_len)); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.by_ref()) { xof_many_equal_len_group::<2>( group_inputs, out_len, @@ -637,7 +640,6 @@ impl AsconXof { aarch64::permute_12_aarch64_neon_x2, ); } - let rem_inputs = input_groups.remainder(); let rem_outputs = output_groups.into_remainder(); if !rem_inputs.is_empty() { Self::hash_many_into_with_kernel( @@ -650,9 +652,9 @@ impl AsconXof { } #[cfg(target_arch = "x86_64")] kernels::AsconPermute12KernelId::X86Avx2 => { - let mut input_groups = inputs.chunks_exact(4); - let mut output_groups = outputs.chunks_exact_mut(4 * out_len); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<4>(); + let mut output_groups = outputs.chunks_exact_mut(4usize.strict_mul(out_len)); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.by_ref()) { xof_many_equal_len_group::<4>( group_inputs, out_len, @@ -661,7 +663,6 @@ impl AsconXof { x86_64_avx2::permute_12_x86_avx2_x4, ); } - let rem_inputs = input_groups.remainder(); let rem_outputs = output_groups.into_remainder(); if !rem_inputs.is_empty() { Self::hash_many_into_with_kernel( @@ -674,9 +675,9 @@ impl AsconXof { } #[cfg(target_arch = "x86_64")] kernels::AsconPermute12KernelId::X86Avx512 => { - let mut input_groups = inputs.chunks_exact(8); - let mut output_groups = outputs.chunks_exact_mut(8 * out_len); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<8>(); + let mut output_groups = outputs.chunks_exact_mut(8usize.strict_mul(out_len)); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.by_ref()) { xof_many_equal_len_group::<8>( group_inputs, out_len, @@ -685,7 +686,6 @@ impl AsconXof { x86_64_avx512::permute_12_x86_avx512_x8, ); } - let rem_inputs = input_groups.remainder(); let rem_outputs = output_groups.into_remainder(); if !rem_inputs.is_empty() { Self::hash_many_into_with_kernel( @@ -709,21 +709,24 @@ impl AsconXof { pub fn hash_many_into(inputs: &[&[u8]], out_len: usize, outputs: &mut [u8]) { assert_eq!( outputs.len(), - inputs.len() * out_len, + inputs.len().strict_mul(out_len), "input/output batch length mismatch" ); + if inputs.is_empty() || out_len == 0 { + return; + } let mut start = 0usize; while start < inputs.len() { let len = inputs[start].len(); - let mut end = start + 1; + let mut end = start.strict_add(1); while end < inputs.len() && inputs[end].len() == len { - end += 1; + end = end.strict_add(1); } - let kid = AsconHash256::batch_kernel_id_for_count(end - start); - let start_byte = start * out_len; - let end_byte = end * out_len; + let kid = AsconHash256::batch_kernel_id_for_count(end.strict_sub(start)); + let start_byte = start.strict_mul(out_len); + let end_byte = end.strict_mul(out_len); Self::hash_many_into_with_kernel(kid, &inputs[start..end], out_len, &mut outputs[start_byte..end_byte]); start = end; } @@ -766,7 +769,7 @@ impl Xof for AsconXofReader { #[inline(always)] fn squeeze(&mut self, mut out: &mut [u8]) { if self.pos < RATE && !out.is_empty() { - let take = core::cmp::min(RATE - self.pos, out.len()); + let take = core::cmp::min(RATE.strict_sub(self.pos), out.len()); out[..take].copy_from_slice(&self.buf[self.pos..self.pos.strict_add(take)]); self.pos = self.pos.strict_add(take); out = &mut out[take..]; diff --git a/src/hashes/crypto/ascon/aarch64.rs b/src/hashes/crypto/ascon/aarch64.rs index a6de44a7..8708590c 100644 --- a/src/hashes/crypto/ascon/aarch64.rs +++ b/src/hashes/crypto/ascon/aarch64.rs @@ -19,7 +19,11 @@ macro_rules! ror_u64x2 { } #[cfg(target_arch = "aarch64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] +/// Combine two scalar words into the low and high lanes of one NEON register. +/// +/// # Safety +/// +/// The caller must ensure the current CPU supports NEON. #[target_feature(enable = "neon")] unsafe fn combine_lanes(a: u64, b: u64) -> uint64x2_t { vcombine_u64(vcreate_u64(a), vcreate_u64(b)) @@ -99,7 +103,6 @@ pub(crate) fn permute_12_aarch64_neon(state: &mut [u64; 5]) { /// /// Caller must ensure the `neon` CPU feature is available. #[cfg(target_arch = "aarch64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] #[target_feature(enable = "neon")] #[inline] unsafe fn permute_12_aarch64_neon_x2_impl(states: &mut [[u64; 5]; 2]) { @@ -167,7 +170,6 @@ unsafe fn permute_12_aarch64_neon_x2_impl(states: &mut [[u64; 5]; 2]) { /// Apply the Ascon-p[12] permutation to two independent states in parallel. #[cfg(target_arch = "aarch64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] #[inline] pub(crate) fn permute_12_aarch64_neon_x2(states: &mut [[u64; 5]; 2]) { // SAFETY: Dispatch verifies aarch64::NEON before selecting this kernel. diff --git a/src/hashes/crypto/ascon/dispatch.rs b/src/hashes/crypto/ascon/dispatch.rs index c915b504..167f3c22 100644 --- a/src/hashes/crypto/ascon/dispatch.rs +++ b/src/hashes/crypto/ascon/dispatch.rs @@ -1,13 +1,17 @@ -#![cfg_attr(test, allow(dead_code))] - +#[cfg(feature = "diag")] +use super::kernels::permute_fn; use super::{ dispatch_tables::DispatchTable, - kernels::{AsconPermute12KernelId, permute_fn, required_caps}, + kernels::{AsconPermute12KernelId, required_caps}, }; -use crate::{backend::cache::OnceCache, platform::Caps}; +#[cfg(feature = "diag")] +use crate::backend::cache::OnceCache; +use crate::platform::Caps; +#[cfg(feature = "diag")] type PermuteFn = fn(&mut [u64; 5]); +#[cfg(feature = "diag")] #[derive(Clone, Copy)] struct ActiveDispatch { boundaries: [usize; 3], @@ -21,6 +25,7 @@ struct ActiveDispatch { l_name: &'static str, } +#[cfg(feature = "diag")] static ACTIVE: OnceCache = OnceCache::new(); #[inline] @@ -33,6 +38,7 @@ fn resolve(id: AsconPermute12KernelId, caps: Caps) -> AsconPermute12KernelId { } } +#[cfg(feature = "diag")] #[inline] #[must_use] fn active() -> ActiveDispatch { @@ -59,6 +65,7 @@ fn active() -> ActiveDispatch { }) } +#[cfg(feature = "diag")] #[inline] #[must_use] fn select(d: &ActiveDispatch, len: usize) -> (PermuteFn, &'static str) { @@ -74,29 +81,14 @@ fn select(d: &ActiveDispatch, len: usize) -> (PermuteFn, &'static str) { } } -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] #[inline] #[must_use] -pub fn kernel_name_for_len(len: usize) -> &'static str { +pub(crate) fn kernel_name_for_len(len: usize) -> &'static str { let d = active(); select(&d, len).1 } -/// Apply the configured Ascon permutation kernel for a specific workload size. -/// -/// The `len` parameter is a hint representing the total amount of work (bytes) -/// associated with the sponge operation. This allows tuned size-class tables to -/// take effect for one-shot and long-running streaming workloads. -/// -/// Production code uses `InlinePermuter` directly; this is retained for the -/// test/bench harness. -#[inline] -#[allow(dead_code)] -pub fn permute_12_for_len(state: &mut [u64; 5], len: usize) { - let d = active(); - (select(&d, len).0)(state); -} - #[cfg(any(test, feature = "std"))] #[inline] #[must_use] diff --git a/src/hashes/crypto/ascon/dispatch_tables.rs b/src/hashes/crypto/ascon/dispatch_tables.rs index 2e507958..7d102ba9 100644 --- a/src/hashes/crypto/ascon/dispatch_tables.rs +++ b/src/hashes/crypto/ascon/dispatch_tables.rs @@ -2,85 +2,88 @@ //! //! This table controls which `permute_12` kernel is used by Ascon hash and XOF. -pub use super::kernels::AsconPermute12KernelId as KernelId; +pub(crate) use super::kernels::AsconPermute12KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +#[cfg(feature = "diag")] +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { + #[cfg(feature = "diag")] pub boundaries: [usize; 3], pub xs: KernelId, + #[cfg(feature = "diag")] pub s: KernelId, + #[cfg(feature = "diag")] pub m: KernelId, + #[cfg(feature = "diag")] pub l: KernelId, } -impl DispatchTable { - #[allow(dead_code)] - #[inline] - #[must_use] - pub const fn kernel_for_len(&self, len: usize) -> KernelId { - let [xs_max, s_max, m_max] = self.boundaries; - if len <= xs_max { - self.xs - } else if len <= s_max { - self.s - } else if len <= m_max { - self.m - } else { - self.l - } - } -} - -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { + #[cfg(feature = "diag")] boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, + #[cfg(feature = "diag")] s: KernelId::Portable, + #[cfg(feature = "diag")] m: KernelId::Portable, + #[cfg(feature = "diag")] l: KernelId::Portable, }; #[cfg(target_arch = "aarch64")] -pub static AARCH64_NEON_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_NEON_TABLE: DispatchTable = DispatchTable { + #[cfg(feature = "diag")] boundaries: DEFAULT_BOUNDARIES, // The single-state policy stays scalar: duplicating each of the five state // words across NEON lanes does not add independent work. The NEON x2 batch // path is wired separately. xs: KernelId::Portable, + #[cfg(feature = "diag")] s: KernelId::Portable, + #[cfg(feature = "diag")] m: KernelId::Portable, + #[cfg(feature = "diag")] l: KernelId::Portable, }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX2_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX2_TABLE: DispatchTable = DispatchTable { + #[cfg(feature = "diag")] boundaries: DEFAULT_BOUNDARIES, // The single-state policy stays scalar: broadcasting each state word across // four AVX2 lanes does not add independent work. The AVX2 x4 batch path is // wired separately. xs: KernelId::Portable, + #[cfg(feature = "diag")] s: KernelId::Portable, + #[cfg(feature = "diag")] m: KernelId::Portable, + #[cfg(feature = "diag")] l: KernelId::Portable, }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512_TABLE: DispatchTable = DispatchTable { + #[cfg(feature = "diag")] boundaries: DEFAULT_BOUNDARIES, // The single-state policy stays scalar: broadcasting each state word across // eight AVX-512 lanes does not add independent work. The AVX-512 x8 batch // path is wired separately. xs: KernelId::Portable, + #[cfg(feature = "diag")] s: KernelId::Portable, + #[cfg(feature = "diag")] m: KernelId::Portable, + #[cfg(feature = "diag")] l: KernelId::Portable, }; #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { #[cfg(target_arch = "aarch64")] { use crate::platform::caps::aarch64; @@ -100,5 +103,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D } } + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/ascon/kernel_test.rs b/src/hashes/crypto/ascon/kernel_test.rs index b194c5fd..16025888 100644 --- a/src/hashes/crypto/ascon/kernel_test.rs +++ b/src/hashes/crypto/ascon/kernel_test.rs @@ -3,9 +3,7 @@ use alloc::{vec, vec::Vec}; use super::kernels::{ALL, permute_fn, required_caps}; #[derive(Clone, Debug)] -#[allow(dead_code)] -pub struct KernelResult { - pub name: &'static str, +pub(super) struct KernelResult { pub state: [u64; 5], } @@ -25,7 +23,7 @@ fn state_from_bytes(data: &[u8]) -> [u64; 5] { } #[must_use] -pub fn run_all_ascon_p12_kernels(data: &[u8]) -> Vec { +pub(super) fn run_all_ascon_p12_kernels(data: &[u8]) -> Vec { let caps = crate::platform::caps(); let mut out = Vec::with_capacity(ALL.len()); let init = state_from_bytes(data); @@ -36,16 +34,13 @@ pub fn run_all_ascon_p12_kernels(data: &[u8]) -> Vec { } let mut st = init; (permute_fn(id))(&mut st); - out.push(KernelResult { - name: id.as_str(), - state: st, - }); + out.push(KernelResult { state: st }); } out } -pub fn verify_ascon_p12_kernels(data: &[u8]) -> Result<(), &'static str> { +pub(super) fn verify_ascon_p12_kernels(data: &[u8]) -> Result<(), &'static str> { let results = run_all_ascon_p12_kernels(data); let Some(first) = results.first() else { return Ok(()); @@ -106,7 +101,7 @@ mod tests { for (input, actual) in inputs.iter().zip(batch.iter()) { let expected = crate::hashes::crypto::AsconHash256::digest(input); - assert_eq!(*actual, expected, "digest_many mismatch for {}", id.as_str()); + assert_eq!(*actual, expected, "digest_many mismatch for {id:?}"); } } } @@ -138,8 +133,7 @@ mod tests { assert_eq!( &batch[base..base + out_len], expected.as_slice(), - "xof_many mismatch for {}", - id.as_str() + "xof_many mismatch for {id:?}" ); } } @@ -188,4 +182,16 @@ mod tests { assert_eq!(&actual[base..base + out_len], expected.as_slice()); } } + + #[test] + fn xof_many_accepts_zero_length_outputs() { + let input = b"same-length input".as_slice(); + let inputs = [input; 8]; + let mut output = []; + + crate::hashes::crypto::AsconXof::hash_many_into(&inputs, 0, &mut output); + for &id in ALL { + crate::hashes::crypto::AsconXof::hash_many_into_with_kernel(id, &inputs, 0, &mut output); + } + } } diff --git a/src/hashes/crypto/ascon/kernels.rs b/src/hashes/crypto/ascon/kernels.rs index 39e21c2c..b088b15e 100644 --- a/src/hashes/crypto/ascon/kernels.rs +++ b/src/hashes/crypto/ascon/kernels.rs @@ -4,8 +4,7 @@ use crate::platform::Caps; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] -pub enum AsconPermute12KernelId { +pub(crate) enum AsconPermute12KernelId { Portable = 0, #[cfg(target_arch = "aarch64")] Aarch64Neon = 1, @@ -16,9 +15,10 @@ pub enum AsconPermute12KernelId { } impl AsconPermute12KernelId { + #[cfg(feature = "diag")] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "aarch64")] @@ -33,8 +33,7 @@ impl AsconPermute12KernelId { #[inline] #[must_use] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] -pub const fn simd_degree(id: AsconPermute12KernelId) -> usize { +pub(crate) const fn simd_degree(id: AsconPermute12KernelId) -> usize { match id { AsconPermute12KernelId::Portable => 1, #[cfg(target_arch = "aarch64")] @@ -47,7 +46,7 @@ pub const fn simd_degree(id: AsconPermute12KernelId) -> usize { } #[must_use] -pub fn permute_fn(id: AsconPermute12KernelId) -> fn(&mut [u64; 5]) { +pub(crate) fn permute_fn(id: AsconPermute12KernelId) -> fn(&mut [u64; 5]) { match id { AsconPermute12KernelId::Portable => permute_12_portable, #[cfg(target_arch = "aarch64")] @@ -61,7 +60,7 @@ pub fn permute_fn(id: AsconPermute12KernelId) -> fn(&mut [u64; 5]) { #[inline] #[must_use] -pub const fn required_caps(id: AsconPermute12KernelId) -> Caps { +pub(crate) const fn required_caps(id: AsconPermute12KernelId) -> Caps { match id { AsconPermute12KernelId::Portable => Caps::NONE, #[cfg(target_arch = "aarch64")] @@ -76,7 +75,7 @@ pub const fn required_caps(id: AsconPermute12KernelId) -> Caps { } #[cfg(test)] -pub const ALL: &[AsconPermute12KernelId] = &[ +pub(crate) const ALL: &[AsconPermute12KernelId] = &[ AsconPermute12KernelId::Portable, #[cfg(target_arch = "aarch64")] AsconPermute12KernelId::Aarch64Neon, diff --git a/src/hashes/crypto/ascon/x86_64_avx2.rs b/src/hashes/crypto/ascon/x86_64_avx2.rs index 6e790905..a2b4fd27 100644 --- a/src/hashes/crypto/ascon/x86_64_avx2.rs +++ b/src/hashes/crypto/ascon/x86_64_avx2.rs @@ -26,15 +26,15 @@ macro_rules! ror_epi64 { #[target_feature(enable = "avx2")] #[inline] unsafe fn permute_12_x86_avx2_impl(state: &mut [u64; 5]) { - let mut x0 = _mm256_set1_epi64x(state[0] as i64); - let mut x1 = _mm256_set1_epi64x(state[1] as i64); - let mut x2 = _mm256_set1_epi64x(state[2] as i64); - let mut x3 = _mm256_set1_epi64x(state[3] as i64); - let mut x4 = _mm256_set1_epi64x(state[4] as i64); + let mut x0 = _mm256_set1_epi64x(state[0].cast_signed()); + let mut x1 = _mm256_set1_epi64x(state[1].cast_signed()); + let mut x2 = _mm256_set1_epi64x(state[2].cast_signed()); + let mut x3 = _mm256_set1_epi64x(state[3].cast_signed()); + let mut x4 = _mm256_set1_epi64x(state[4].cast_signed()); let ones = _mm256_set1_epi64x(-1); for &c in &super::RC { - x2 = _mm256_xor_si256(x2, _mm256_set1_epi64x(c as i64)); + x2 = _mm256_xor_si256(x2, _mm256_set1_epi64x(c.cast_signed())); x0 = _mm256_xor_si256(x0, x4); x4 = _mm256_xor_si256(x4, x3); @@ -70,11 +70,11 @@ unsafe fn permute_12_x86_avx2_impl(state: &mut [u64; 5]) { x4 = _mm256_xor_si256(y4, _mm256_xor_si256(ror_epi64!(y4, 7, 57), ror_epi64!(y4, 41, 23))); } - state[0] = _mm256_extract_epi64::<0>(x0) as u64; - state[1] = _mm256_extract_epi64::<0>(x1) as u64; - state[2] = _mm256_extract_epi64::<0>(x2) as u64; - state[3] = _mm256_extract_epi64::<0>(x3) as u64; - state[4] = _mm256_extract_epi64::<0>(x4) as u64; + state[0] = _mm256_extract_epi64::<0>(x0).cast_unsigned(); + state[1] = _mm256_extract_epi64::<0>(x1).cast_unsigned(); + state[2] = _mm256_extract_epi64::<0>(x2).cast_unsigned(); + state[3] = _mm256_extract_epi64::<0>(x3).cast_unsigned(); + state[4] = _mm256_extract_epi64::<0>(x4).cast_unsigned(); } /// Apply the Ascon-p[12] permutation using x86_64 AVX2. @@ -91,44 +91,43 @@ pub(crate) fn permute_12_x86_avx2(state: &mut [u64; 5]) { /// /// Caller must ensure the `avx2` CPU feature is available. #[cfg(target_arch = "x86_64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] #[target_feature(enable = "avx2")] #[inline] unsafe fn permute_12_x86_avx2_x4_impl(states: &mut [[u64; 5]; 4]) { let mut x0 = _mm256_set_epi64x( - states[3][0] as i64, - states[2][0] as i64, - states[1][0] as i64, - states[0][0] as i64, + states[3][0].cast_signed(), + states[2][0].cast_signed(), + states[1][0].cast_signed(), + states[0][0].cast_signed(), ); let mut x1 = _mm256_set_epi64x( - states[3][1] as i64, - states[2][1] as i64, - states[1][1] as i64, - states[0][1] as i64, + states[3][1].cast_signed(), + states[2][1].cast_signed(), + states[1][1].cast_signed(), + states[0][1].cast_signed(), ); let mut x2 = _mm256_set_epi64x( - states[3][2] as i64, - states[2][2] as i64, - states[1][2] as i64, - states[0][2] as i64, + states[3][2].cast_signed(), + states[2][2].cast_signed(), + states[1][2].cast_signed(), + states[0][2].cast_signed(), ); let mut x3 = _mm256_set_epi64x( - states[3][3] as i64, - states[2][3] as i64, - states[1][3] as i64, - states[0][3] as i64, + states[3][3].cast_signed(), + states[2][3].cast_signed(), + states[1][3].cast_signed(), + states[0][3].cast_signed(), ); let mut x4 = _mm256_set_epi64x( - states[3][4] as i64, - states[2][4] as i64, - states[1][4] as i64, - states[0][4] as i64, + states[3][4].cast_signed(), + states[2][4].cast_signed(), + states[1][4].cast_signed(), + states[0][4].cast_signed(), ); let ones = _mm256_set1_epi64x(-1); for &c in &super::RC { - x2 = _mm256_xor_si256(x2, _mm256_set1_epi64x(c as i64)); + x2 = _mm256_xor_si256(x2, _mm256_set1_epi64x(c.cast_signed())); x0 = _mm256_xor_si256(x0, x4); x4 = _mm256_xor_si256(x4, x3); @@ -164,31 +163,30 @@ unsafe fn permute_12_x86_avx2_x4_impl(states: &mut [[u64; 5]; 4]) { x4 = _mm256_xor_si256(y4, _mm256_xor_si256(ror_epi64!(y4, 7, 57), ror_epi64!(y4, 41, 23))); } - states[0][0] = _mm256_extract_epi64::<0>(x0) as u64; - states[1][0] = _mm256_extract_epi64::<1>(x0) as u64; - states[2][0] = _mm256_extract_epi64::<2>(x0) as u64; - states[3][0] = _mm256_extract_epi64::<3>(x0) as u64; - states[0][1] = _mm256_extract_epi64::<0>(x1) as u64; - states[1][1] = _mm256_extract_epi64::<1>(x1) as u64; - states[2][1] = _mm256_extract_epi64::<2>(x1) as u64; - states[3][1] = _mm256_extract_epi64::<3>(x1) as u64; - states[0][2] = _mm256_extract_epi64::<0>(x2) as u64; - states[1][2] = _mm256_extract_epi64::<1>(x2) as u64; - states[2][2] = _mm256_extract_epi64::<2>(x2) as u64; - states[3][2] = _mm256_extract_epi64::<3>(x2) as u64; - states[0][3] = _mm256_extract_epi64::<0>(x3) as u64; - states[1][3] = _mm256_extract_epi64::<1>(x3) as u64; - states[2][3] = _mm256_extract_epi64::<2>(x3) as u64; - states[3][3] = _mm256_extract_epi64::<3>(x3) as u64; - states[0][4] = _mm256_extract_epi64::<0>(x4) as u64; - states[1][4] = _mm256_extract_epi64::<1>(x4) as u64; - states[2][4] = _mm256_extract_epi64::<2>(x4) as u64; - states[3][4] = _mm256_extract_epi64::<3>(x4) as u64; + states[0][0] = _mm256_extract_epi64::<0>(x0).cast_unsigned(); + states[1][0] = _mm256_extract_epi64::<1>(x0).cast_unsigned(); + states[2][0] = _mm256_extract_epi64::<2>(x0).cast_unsigned(); + states[3][0] = _mm256_extract_epi64::<3>(x0).cast_unsigned(); + states[0][1] = _mm256_extract_epi64::<0>(x1).cast_unsigned(); + states[1][1] = _mm256_extract_epi64::<1>(x1).cast_unsigned(); + states[2][1] = _mm256_extract_epi64::<2>(x1).cast_unsigned(); + states[3][1] = _mm256_extract_epi64::<3>(x1).cast_unsigned(); + states[0][2] = _mm256_extract_epi64::<0>(x2).cast_unsigned(); + states[1][2] = _mm256_extract_epi64::<1>(x2).cast_unsigned(); + states[2][2] = _mm256_extract_epi64::<2>(x2).cast_unsigned(); + states[3][2] = _mm256_extract_epi64::<3>(x2).cast_unsigned(); + states[0][3] = _mm256_extract_epi64::<0>(x3).cast_unsigned(); + states[1][3] = _mm256_extract_epi64::<1>(x3).cast_unsigned(); + states[2][3] = _mm256_extract_epi64::<2>(x3).cast_unsigned(); + states[3][3] = _mm256_extract_epi64::<3>(x3).cast_unsigned(); + states[0][4] = _mm256_extract_epi64::<0>(x4).cast_unsigned(); + states[1][4] = _mm256_extract_epi64::<1>(x4).cast_unsigned(); + states[2][4] = _mm256_extract_epi64::<2>(x4).cast_unsigned(); + states[3][4] = _mm256_extract_epi64::<3>(x4).cast_unsigned(); } /// Apply the Ascon-p[12] permutation to four independent states in parallel. #[cfg(target_arch = "x86_64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] #[inline] pub(crate) fn permute_12_x86_avx2_x4(states: &mut [[u64; 5]; 4]) { // SAFETY: Dispatch verifies x86::AVX2 before selecting this kernel. diff --git a/src/hashes/crypto/ascon/x86_64_avx512.rs b/src/hashes/crypto/ascon/x86_64_avx512.rs index 06c52167..a6e96731 100644 --- a/src/hashes/crypto/ascon/x86_64_avx512.rs +++ b/src/hashes/crypto/ascon/x86_64_avx512.rs @@ -31,15 +31,15 @@ macro_rules! ror_epi64x8 { #[target_feature(enable = "avx512f,avx512vl")] #[inline] unsafe fn permute_12_x86_avx512_impl(state: &mut [u64; 5]) { - let mut x0 = _mm256_set1_epi64x(state[0] as i64); - let mut x1 = _mm256_set1_epi64x(state[1] as i64); - let mut x2 = _mm256_set1_epi64x(state[2] as i64); - let mut x3 = _mm256_set1_epi64x(state[3] as i64); - let mut x4 = _mm256_set1_epi64x(state[4] as i64); + let mut x0 = _mm256_set1_epi64x(state[0].cast_signed()); + let mut x1 = _mm256_set1_epi64x(state[1].cast_signed()); + let mut x2 = _mm256_set1_epi64x(state[2].cast_signed()); + let mut x3 = _mm256_set1_epi64x(state[3].cast_signed()); + let mut x4 = _mm256_set1_epi64x(state[4].cast_signed()); let ones = _mm256_set1_epi64x(-1); for &c in &super::RC { - x2 = _mm256_xor_si256(x2, _mm256_set1_epi64x(c as i64)); + x2 = _mm256_xor_si256(x2, _mm256_set1_epi64x(c.cast_signed())); x0 = _mm256_xor_si256(x0, x4); x4 = _mm256_xor_si256(x4, x3); @@ -75,11 +75,11 @@ unsafe fn permute_12_x86_avx512_impl(state: &mut [u64; 5]) { x4 = _mm256_xor_si256(y4, _mm256_xor_si256(ror_epi64!(y4, 57), ror_epi64!(y4, 23))); } - state[0] = _mm256_extract_epi64::<0>(x0) as u64; - state[1] = _mm256_extract_epi64::<0>(x1) as u64; - state[2] = _mm256_extract_epi64::<0>(x2) as u64; - state[3] = _mm256_extract_epi64::<0>(x3) as u64; - state[4] = _mm256_extract_epi64::<0>(x4) as u64; + state[0] = _mm256_extract_epi64::<0>(x0).cast_unsigned(); + state[1] = _mm256_extract_epi64::<0>(x1).cast_unsigned(); + state[2] = _mm256_extract_epi64::<0>(x2).cast_unsigned(); + state[3] = _mm256_extract_epi64::<0>(x3).cast_unsigned(); + state[4] = _mm256_extract_epi64::<0>(x4).cast_unsigned(); } /// Apply the Ascon-p[12] permutation using x86_64 AVX-512. @@ -96,64 +96,63 @@ pub(crate) fn permute_12_x86_avx512(state: &mut [u64; 5]) { /// /// Caller must ensure the `avx512f` and `avx512vl` CPU features are available. #[cfg(target_arch = "x86_64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] #[target_feature(enable = "avx512f,avx512vl")] #[inline] unsafe fn permute_12_x86_avx512_x8_impl(states: &mut [[u64; 5]; 8]) { let mut x0 = _mm512_set_epi64( - states[7][0] as i64, - states[6][0] as i64, - states[5][0] as i64, - states[4][0] as i64, - states[3][0] as i64, - states[2][0] as i64, - states[1][0] as i64, - states[0][0] as i64, + states[7][0].cast_signed(), + states[6][0].cast_signed(), + states[5][0].cast_signed(), + states[4][0].cast_signed(), + states[3][0].cast_signed(), + states[2][0].cast_signed(), + states[1][0].cast_signed(), + states[0][0].cast_signed(), ); let mut x1 = _mm512_set_epi64( - states[7][1] as i64, - states[6][1] as i64, - states[5][1] as i64, - states[4][1] as i64, - states[3][1] as i64, - states[2][1] as i64, - states[1][1] as i64, - states[0][1] as i64, + states[7][1].cast_signed(), + states[6][1].cast_signed(), + states[5][1].cast_signed(), + states[4][1].cast_signed(), + states[3][1].cast_signed(), + states[2][1].cast_signed(), + states[1][1].cast_signed(), + states[0][1].cast_signed(), ); let mut x2 = _mm512_set_epi64( - states[7][2] as i64, - states[6][2] as i64, - states[5][2] as i64, - states[4][2] as i64, - states[3][2] as i64, - states[2][2] as i64, - states[1][2] as i64, - states[0][2] as i64, + states[7][2].cast_signed(), + states[6][2].cast_signed(), + states[5][2].cast_signed(), + states[4][2].cast_signed(), + states[3][2].cast_signed(), + states[2][2].cast_signed(), + states[1][2].cast_signed(), + states[0][2].cast_signed(), ); let mut x3 = _mm512_set_epi64( - states[7][3] as i64, - states[6][3] as i64, - states[5][3] as i64, - states[4][3] as i64, - states[3][3] as i64, - states[2][3] as i64, - states[1][3] as i64, - states[0][3] as i64, + states[7][3].cast_signed(), + states[6][3].cast_signed(), + states[5][3].cast_signed(), + states[4][3].cast_signed(), + states[3][3].cast_signed(), + states[2][3].cast_signed(), + states[1][3].cast_signed(), + states[0][3].cast_signed(), ); let mut x4 = _mm512_set_epi64( - states[7][4] as i64, - states[6][4] as i64, - states[5][4] as i64, - states[4][4] as i64, - states[3][4] as i64, - states[2][4] as i64, - states[1][4] as i64, - states[0][4] as i64, + states[7][4].cast_signed(), + states[6][4].cast_signed(), + states[5][4].cast_signed(), + states[4][4].cast_signed(), + states[3][4].cast_signed(), + states[2][4].cast_signed(), + states[1][4].cast_signed(), + states[0][4].cast_signed(), ); let ones = _mm512_set1_epi64(-1); for &c in &super::RC { - x2 = _mm512_xor_si512(x2, _mm512_set1_epi64(c as i64)); + x2 = _mm512_xor_si512(x2, _mm512_set1_epi64(c.cast_signed())); x0 = _mm512_xor_si512(x0, x4); x4 = _mm512_xor_si512(x4, x3); @@ -213,7 +212,6 @@ unsafe fn permute_12_x86_avx512_x8_impl(states: &mut [[u64; 5]; 8]) { /// Apply the Ascon-p[12] permutation to eight independent states in parallel. #[cfg(target_arch = "x86_64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] #[inline] pub(crate) fn permute_12_x86_avx512_x8(states: &mut [[u64; 5]; 8]) { // SAFETY: Dispatch verifies x86::AVX512F + x86::AVX512VL before selecting this kernel. diff --git a/src/hashes/crypto/blake2b/kernels.rs b/src/hashes/crypto/blake2b/kernels.rs index 6996ebf4..73f342b2 100644 --- a/src/hashes/crypto/blake2b/kernels.rs +++ b/src/hashes/crypto/blake2b/kernels.rs @@ -1,5 +1,6 @@ //! Blake2b portable compression function and kernel dispatch (RFC 7693). +#[cfg(any(test, not(all(target_arch = "aarch64", target_os = "macos"))))] use crate::platform::Caps; #[cfg(target_arch = "riscv64")] use crate::platform::caps::riscv; @@ -7,6 +8,7 @@ use crate::platform::caps::riscv; use crate::platform::caps::wasm; #[cfg(target_arch = "x86_64")] use crate::platform::caps::x86; +use crate::traits::ct; /// Blake2b compress function pointer type. /// @@ -50,14 +52,19 @@ impl Blake2bCounter { pub(crate) const fn as_u128(self) -> u128 { ((self.hi as u128) << 64) | self.lo as u128 } + + #[inline(always)] + pub(crate) fn zeroize_no_fence(&mut self) { + ct::zeroize_words_no_fence(core::slice::from_mut(&mut self.lo)); + ct::zeroize_words_no_fence(core::slice::from_mut(&mut self.hi)); + } } /// Blake2b kernel identifier. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -#[cfg_attr(target_os = "macos", allow(dead_code))] -pub enum Blake2bKernelId { +pub(crate) enum Blake2bKernelId { Portable = 0, #[cfg(target_arch = "x86_64")] X86Avx2 = 1, @@ -73,7 +80,7 @@ impl Blake2bKernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "x86_64")] @@ -115,14 +122,12 @@ fn compress_wasm_simd128(h: &mut [u64; 8], block: &[u8; 128], t: u128, last: boo #[inline(always)] fn compress_blocks_with(h: &mut [u64; 8], blocks: &[u8], t: &mut Blake2bCounter, compress: CompressFn) { debug_assert_eq!(blocks.len() % 128, 0); - let mut chunks = blocks.chunks_exact(128); - for chunk in &mut chunks { + let (chunks, remainder) = blocks.as_chunks::<128>(); + for block in chunks { t.add_len(128); - // SAFETY: `chunks_exact(128)` yields slices of exactly 128 bytes. - let block = unsafe { &*chunk.as_ptr().cast::<[u8; 128]>() }; compress(h, block, t.as_u128(), false); } - debug_assert!(chunks.remainder().is_empty()); + debug_assert!(remainder.is_empty()); } fn compress_blocks_portable(h: &mut [u64; 8], blocks: &[u8], t: &mut Blake2bCounter) { @@ -181,10 +186,10 @@ pub(crate) fn compress_blocks_fn(id: Blake2bKernelId) -> CompressBlocksFn { } /// Capabilities required to run the given kernel. +#[cfg(any(test, not(all(target_arch = "aarch64", target_os = "macos"))))] #[inline] #[must_use] -#[allow(dead_code)] // Used by runtime dispatch on targets that don't bypass to a fixed kernel. -pub const fn required_caps(id: Blake2bKernelId) -> Caps { +pub(crate) const fn required_caps(id: Blake2bKernelId) -> Caps { match id { Blake2bKernelId::Portable => Caps::NONE, #[cfg(target_arch = "x86_64")] @@ -200,7 +205,7 @@ pub const fn required_caps(id: Blake2bKernelId) -> Caps { /// All kernel IDs for agreement testing. #[cfg(test)] -pub const ALL: &[Blake2bKernelId] = &[ +pub(crate) const ALL: &[Blake2bKernelId] = &[ Blake2bKernelId::Portable, #[cfg(target_arch = "x86_64")] Blake2bKernelId::X86Avx2, @@ -237,7 +242,6 @@ pub(crate) fn compile_time_best() -> CompressFn { { return compress_x86_avx2; } - #[allow(unreachable_code)] compress } @@ -255,7 +259,6 @@ pub(crate) fn compile_time_best_blocks() -> CompressBlocksFn { { return compress_blocks_x86_avx2; } - #[allow(unreachable_code)] compress_blocks_portable } @@ -272,7 +275,6 @@ pub(crate) const IV: [u64; 8] = [ ]; /// Message-word permutation schedule (10 rows, reused cyclically for 12 rounds). -#[allow(dead_code)] // Used by target-specific SIMD backends that are not compiled on every host. pub(crate) const SIGMA: [[u8; 16]; 10] = [ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3], @@ -426,7 +428,6 @@ fn round(v: &mut [U64x4; 4], m: &[u64; 16], s: &[u8; 16]) { /// Load 16 little-endian u64 message words from a 128-byte block. #[inline(always)] -#[allow(clippy::indexing_slicing)] // i is always in 0..16 (loop bound). pub(crate) fn load_msg(block: &[u8; 128]) -> [u64; 16] { let mut m = [0u64; 16]; let src = block.as_ptr(); @@ -439,18 +440,28 @@ pub(crate) fn load_msg(block: &[u8; 128]) -> [u64; 16] { m } +#[inline(always)] +fn split_counter(counter: u128) -> (u64, u64) { + let [b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15] = counter.to_le_bytes(); + ( + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]), + u64::from_le_bytes([b8, b9, b10, b11, b12, b13, b14, b15]), + ) +} + /// Initialize the 16-word working vector from state, IV, counter, and finalization flag. #[cfg(any(target_arch = "x86_64", target_arch = "wasm32", target_arch = "riscv64"))] #[inline(always)] pub(crate) fn init_v(h: &[u64; 8], t: u128, last: bool) -> [u64; 16] { + let (t0, t1) = split_counter(t); let mut v = [0u64; 16]; v[..8].copy_from_slice(h); v[8] = IV[0]; v[9] = IV[1]; v[10] = IV[2]; v[11] = IV[3]; - v[12] = IV[4] ^ (t as u64); - v[13] = IV[5] ^ ((t >> 64) as u64); + v[12] = IV[4] ^ t0; + v[13] = IV[5] ^ t1; v[14] = if last { IV[6] ^ u64::MAX } else { IV[6] }; v[15] = IV[7]; v @@ -460,11 +471,9 @@ pub(crate) fn init_v(h: &[u64; 8], t: u128, last: bool) -> [u64; 16] { /// /// `t` is the total number of input bytes after this block (inclusive). /// `last` is `true` for the final block (sets the finalization flag). -#[allow(clippy::indexing_slicing)] pub(crate) fn compress(h: &mut [u64; 8], block: &[u8; 128], t: u128, last: bool) { let m = load_msg(block); - let t0 = t as u64; - let t1 = (t >> 64) as u64; + let (t0, t1) = split_counter(t); let f0 = if last { u64::MAX } else { 0 }; let mut v = [ diff --git a/src/hashes/crypto/blake2b/mod.rs b/src/hashes/crypto/blake2b/mod.rs index b56fad26..a54d46f8 100644 --- a/src/hashes/crypto/blake2b/mod.rs +++ b/src/hashes/crypto/blake2b/mod.rs @@ -65,9 +65,12 @@ use kernels::{Blake2bCounter, IV}; use super::Blake2Error; use crate::traits::{Digest, ct}; -const BLOCK_SIZE: usize = 128; -const MAX_KEY_LEN: usize = 64; -const MAX_OUTPUT_LEN: usize = 64; +const BLOCK_SIZE_U8: u8 = 128; +const BLOCK_SIZE: usize = BLOCK_SIZE_U8 as usize; +const MAX_KEY_LEN_U8: u8 = 64; +const MAX_KEY_LEN: usize = MAX_KEY_LEN_U8 as usize; +const MAX_OUTPUT_LEN_U8: u8 = 64; +const MAX_OUTPUT_LEN: usize = MAX_OUTPUT_LEN_U8 as usize; /// Validated 1–64 byte Blake2b key. /// @@ -124,14 +127,14 @@ fn validate_key(key: &[u8]) -> Result<(), Blake2Error> { #[inline] fn validate_output_len(output_len: usize) -> Result { - if !(1..=MAX_OUTPUT_LEN).contains(&output_len) { + let output_len = u8::try_from(output_len).map_err(|_| Blake2Error::InvalidOutputLength)?; + if !(1..=MAX_OUTPUT_LEN_U8).contains(&output_len) { return Err(Blake2Error::InvalidOutputLength); } - Ok(output_len as u8) + Ok(output_len) } -#[cfg(any(test, feature = "diag"))] -#[allow(dead_code)] +#[cfg(feature = "diag")] #[inline] #[must_use] pub(crate) fn kernel_name_for_len(len: usize) -> &'static str { @@ -182,7 +185,6 @@ impl Core { /// Create a new Blake2b state with output length `nn`, optional `key`, and /// spec-defined `salt` + `personal` parameter-block values (RFC 7693 §2.5). - #[allow(clippy::indexing_slicing)] fn new_with_params(nn: u8, key: &[u8], salt: &[u8; SALT_LEN], personal: &[u8; PERSONAL_LEN]) -> Self { assert!( nn >= 1 && nn as usize <= MAX_OUTPUT_LEN, @@ -190,7 +192,7 @@ impl Core { ); assert!(key.len() <= MAX_KEY_LEN, "Blake2b key must be at most 64 bytes"); - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2b key length fits in u8"); let h = init_state_with_params(nn, kk, salt, personal); let stored_key = if kk > 0 { @@ -204,7 +206,7 @@ impl Core { let mut buf = [0u8; BLOCK_SIZE]; let buf_len = if kk > 0 { buf[..key.len()].copy_from_slice(key); - BLOCK_SIZE as u8 + BLOCK_SIZE_U8 } else { 0 }; @@ -228,30 +230,23 @@ impl Core { fn zeroize_key_if_any(&mut self) { if self.kk > 0 { // SAFETY: when `kk > 0`, `self.key` was initialized in `new`. - unsafe { ct::zeroize_no_fence(&mut *self.key.as_mut_ptr()) }; + unsafe { ct::zeroize_no_fence(self.key.assume_init_mut()) }; } } #[inline(always)] fn wipe(&mut self) { - for word in self.h.iter_mut() { - // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut self.h); ct::zeroize_no_fence(&mut self.buf); self.zeroize_key_if_any(); - // SAFETY: fields are valid, aligned, dereferenceable pointers. - unsafe { - core::ptr::write_volatile(&mut self.buf_len, 0); - core::ptr::write_volatile(&mut self.t, Blake2bCounter::zero()); - core::ptr::write_volatile(&mut self.nn, 0); - core::ptr::write_volatile(&mut self.kk, 0); - } + ct::zeroize_no_fence(core::slice::from_mut(&mut self.buf_len)); + self.t.zeroize_no_fence(); + ct::zeroize_no_fence(core::slice::from_mut(&mut self.nn)); + ct::zeroize_no_fence(core::slice::from_mut(&mut self.kk)); core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } /// Feed data into the hash state. - #[allow(clippy::indexing_slicing)] fn update(&mut self, data: &[u8]) { if data.is_empty() { return; @@ -265,7 +260,7 @@ impl Core { if self.buf_len > 0 && (self.buf_len as usize).strict_add(data_len) > BLOCK_SIZE { let fill = BLOCK_SIZE.strict_sub(self.buf_len as usize); self.buf[self.buf_len as usize..BLOCK_SIZE].copy_from_slice(&data[..fill]); - self.t.add_len(BLOCK_SIZE as u64); + self.t.add_len(u64::from(BLOCK_SIZE_U8)); (self.compress)(&mut self.h, &self.buf, self.t.as_u128(), false); self.buf_len = 0; offset = fill; @@ -275,7 +270,7 @@ impl Core { // in the buffer so finalize can set the final flag. let available = data_len.strict_sub(offset); if available > BLOCK_SIZE { - let blocks_len = available.strict_sub(1) / BLOCK_SIZE * BLOCK_SIZE; + let blocks_len = (available.strict_sub(1) / BLOCK_SIZE).strict_mul(BLOCK_SIZE); let end = offset.strict_add(blocks_len); (self.compress_blocks)(&mut self.h, &data[offset..end], &mut self.t); offset = end; @@ -286,12 +281,12 @@ impl Core { if remaining > 0 { let start = self.buf_len as usize; self.buf[start..start.strict_add(remaining)].copy_from_slice(&data[offset..]); - self.buf_len = self.buf_len.strict_add(remaining as u8); + let remaining = u8::try_from(remaining).expect("Blake2b buffer remainder fits in u8"); + self.buf_len = self.buf_len.strict_add(remaining); } } /// Finalize and write the hash into `out` (must be exactly `nn` bytes). - #[allow(clippy::indexing_slicing)] fn finalize_into(&self, out: &mut [u8]) { debug_assert!(out.len() == self.nn as usize); @@ -308,16 +303,10 @@ impl Core { write_output(&h, self.nn, out); - for word in h.iter_mut() { - // SAFETY: Volatile zeroization of a stack word is sound because: - // 1. `word` comes from `h.iter_mut()`, so it is valid and uniquely borrowed. - // 2. `u64` has no invalid bit patterns; writing zero preserves validity. - // 3. The pointer does not escape this loop iteration. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut h); } - #[cfg(any(test, feature = "diag"))] + #[cfg(any(test, all(feature = "diag", feature = "argon2")))] fn new_with_compress_for_test( nn: u8, key: &[u8], @@ -330,7 +319,7 @@ impl Core { ); assert!(key.len() <= MAX_KEY_LEN, "Blake2b key must be at most 64 bytes"); - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2b key length fits in u8"); let h = init_state(nn, kk); let stored_key = if kk > 0 { let mut bytes = [0u8; MAX_KEY_LEN]; @@ -346,7 +335,7 @@ impl Core { if let Some(dst) = buf.get_mut(..key.len()) { dst.copy_from_slice(key); } - BLOCK_SIZE as u8 + BLOCK_SIZE_U8 } else { 0 }; @@ -368,19 +357,16 @@ impl Core { /// Reset to the initial state (including re-buffering the key if keyed and /// re-applying any salt/personalization parameter-block bytes). - #[allow(clippy::indexing_slicing)] fn reset(&mut self) { self.h = init_state_with_params(self.nn, self.kk, &self.salt, &self.personal); if self.kk > 0 { let key_len = self.kk as usize; self.buf = [0u8; BLOCK_SIZE]; - // SAFETY: when `kk > 0`, `self.key` was initialized in `new`, and `self.buf` - // has at least `key_len` bytes available. - unsafe { - core::ptr::copy_nonoverlapping(self.key.as_ptr().cast::(), self.buf.as_mut_ptr(), key_len); - } - self.buf_len = BLOCK_SIZE as u8; + // SAFETY: when `kk > 0`, `self.key` was initialized in `new`. + let key = unsafe { self.key.assume_init_ref() }; + self.buf[..key_len].copy_from_slice(&key[..key_len]); + self.buf_len = BLOCK_SIZE_U8; } else { self.buf = [0u8; BLOCK_SIZE]; self.buf_len = 0; @@ -421,7 +407,6 @@ const BLAKE2B512_H0: [u64; 8] = init_unkeyed_fixed_state(64); /// digest length / key length / fanout / depth bits, and `h[4..8]` carry the /// salt and personalization words. #[inline] -#[allow(clippy::indexing_slicing)] fn init_state_with_params(nn: u8, kk: u8, salt: &[u8; SALT_LEN], personal: &[u8; PERSONAL_LEN]) -> [u64; 8] { let p0 = nn as u64 | ((kk as u64) << 8) | (1u64 << 16) | (1u64 << 24); let mut h = IV; @@ -439,7 +424,6 @@ fn init_state_with_params(nn: u8, kk: u8, salt: &[u8; SALT_LEN], personal: &[u8; } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_output(h: &[u64; 8], nn: u8, out: &mut [u8]) { let nn = nn as usize; let full_words = nn / 8; @@ -468,7 +452,6 @@ fn write_output(h: &[u64; 8], nn: u8, out: &mut [u8]) { } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn oneshot_small_into_with_params( nn: u8, key: &[u8], @@ -477,7 +460,7 @@ fn oneshot_small_into_with_params( out: &mut [u8], compress: kernels::CompressFn, ) { - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2b key length fits in u8"); let mut h = match params { Some((salt, personal)) => init_state_with_params(nn, kk, salt, personal), None => init_state(nn, kk), @@ -495,30 +478,26 @@ fn oneshot_small_into_with_params( block[..key.len()].copy_from_slice(key); if data.is_empty() { - compress(&mut h, &block, BLOCK_SIZE as u128, true); + compress(&mut h, &block, u128::from(BLOCK_SIZE_U8), true); ct::zeroize(&mut block); } else { - compress(&mut h, &block, BLOCK_SIZE as u128, false); + compress(&mut h, &block, u128::from(BLOCK_SIZE_U8), false); ct::zeroize(&mut block); block[..data.len()].copy_from_slice(data); compress( &mut h, &block, - (BLOCK_SIZE as u128).strict_add(data.len() as u128), + u128::from(BLOCK_SIZE_U8).strict_add(data.len() as u128), true, ); } write_output(&h, nn, out); - for word in &mut h { - // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut h); } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn oneshot_hash_into_inner( nn: u8, key: &[u8], @@ -539,7 +518,7 @@ fn oneshot_hash_into_inner( return; } - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2b key length fits in u8"); let mut h = match params { Some((salt, personal)) => init_state_with_params(nn, kk, salt, personal), None => init_state(nn, kk), @@ -547,7 +526,7 @@ fn oneshot_hash_into_inner( let mut buf = [0u8; BLOCK_SIZE]; let mut buf_len = if kk > 0 { buf[..key.len()].copy_from_slice(key); - BLOCK_SIZE as u8 + BLOCK_SIZE_U8 } else { 0 }; @@ -560,7 +539,7 @@ fn oneshot_hash_into_inner( if fill > 0 { buf[buf_len as usize..BLOCK_SIZE].copy_from_slice(&data[..fill]); } - t.add_len(BLOCK_SIZE as u64); + t.add_len(u64::from(BLOCK_SIZE_U8)); compress(&mut h, &buf, t.as_u128(), false); ct::zeroize(&mut buf); buf_len = 0; @@ -569,7 +548,7 @@ fn oneshot_hash_into_inner( let available = data_len.strict_sub(offset); if available > BLOCK_SIZE { - let blocks_len = available.strict_sub(1) / BLOCK_SIZE * BLOCK_SIZE; + let blocks_len = (available.strict_sub(1) / BLOCK_SIZE).strict_mul(BLOCK_SIZE); let end = offset.strict_add(blocks_len); let compress_blocks = dispatch::compress_blocks_dispatch(); compress_blocks(&mut h, &data[offset..end], &mut t); @@ -579,7 +558,7 @@ fn oneshot_hash_into_inner( let remaining = data_len.strict_sub(offset); if remaining > 0 { buf[..remaining].copy_from_slice(&data[offset..]); - buf_len = remaining as u8; + buf_len = u8::try_from(remaining).expect("Blake2b buffer remainder fits in u8"); } t.add_len(buf_len as u64); @@ -587,10 +566,7 @@ fn oneshot_hash_into_inner( write_output(&h, nn, out); if kk > 0 { - for word in &mut h { - // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut h); ct::zeroize(&mut buf); } } @@ -707,12 +683,11 @@ impl Blake2bParams { /// Set a validated MAC key. Omit this method for unkeyed hashing. #[must_use] - #[allow(clippy::indexing_slicing)] pub fn key(mut self, key: Blake2bKey<'_>) -> Self { let key = key.as_bytes(); self.key_buf = [0u8; MAX_KEY_LEN]; self.key_buf[..key.len()].copy_from_slice(key); - self.key_len = key.len() as u8; + self.key_len = u8::try_from(key.len()).expect("validated Blake2b key length fits in u8"); self } @@ -730,7 +705,6 @@ impl Blake2bParams { self } - #[allow(clippy::indexing_slicing)] fn key_slice(&self) -> &[u8] { &self.key_buf[..usize::from(self.key_len)] } @@ -877,7 +851,6 @@ impl Blake2b256 { #[must_use] pub fn new_keyed(key: Blake2bKey<'_>) -> Self { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); Self(Core::new(32, key)) } @@ -885,14 +858,12 @@ impl Blake2b256 { #[must_use] pub fn keyed_digest(key: Blake2bKey<'_>, data: &[u8]) -> [u8; 32] { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); oneshot_hash_array::<32>(32, key, data) } } impl Blake2b256 { - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn new_with_compress_for_test( compress: kernels::CompressFn, compress_blocks: kernels::CompressBlocksFn, @@ -900,8 +871,7 @@ impl Blake2b256 { Self(Core::new_with_compress_for_test(32, &[], compress, compress_blocks)) } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn keyed_with_compress_for_test( key: &[u8], compress: kernels::CompressFn, @@ -914,6 +884,7 @@ impl Blake2b256 { #[cfg(feature = "diag")] #[must_use] +/// Derive a keyed Blake2b-256 diagnostic digest with the portable backend. pub fn diag_blake2b256_keyed_digest_portable(key: &[u8; 32]) -> [u8; 32] { let mut out = [0u8; 32]; oneshot_small_into_with_params( @@ -927,7 +898,7 @@ pub fn diag_blake2b256_keyed_digest_portable(key: &[u8; 32]) -> [u8; 32] { out } -#[cfg(any(test, all(feature = "diag", feature = "argon2")))] +#[cfg(all(feature = "diag", feature = "argon2"))] pub(crate) fn diag_hash_parts_portable(output_len: u8, parts: &[&[u8]], out: &mut [u8]) { assert_eq!( out.len(), @@ -1022,7 +993,6 @@ impl Blake2b512 { #[must_use] pub fn new_keyed(key: Blake2bKey<'_>) -> Self { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); Self(Core::new(64, key)) } @@ -1030,14 +1000,12 @@ impl Blake2b512 { #[must_use] pub fn keyed_digest(key: Blake2bKey<'_>, data: &[u8]) -> [u8; 64] { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); oneshot_hash_array::<64>(64, key, data) } } impl Blake2b512 { - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn new_with_compress_for_test( compress: kernels::CompressFn, compress_blocks: kernels::CompressBlocksFn, @@ -1156,7 +1124,6 @@ impl Blake2b { pub fn new_keyed(output_len: usize, key: Blake2bKey<'_>) -> Result { let output_len = validate_output_len(output_len)?; let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); Ok(Self { core: Core::new(output_len, key), output_len, @@ -1222,7 +1189,6 @@ impl Blake2b { pub fn keyed_digest_into(key: Blake2bKey<'_>, data: &[u8], out: &mut [u8]) -> Result<(), Blake2Error> { let output_len = validate_output_len(out.len())?; let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); oneshot_hash_into(output_len, key, data, out); Ok(()) } @@ -1237,7 +1203,8 @@ impl Blake2b { const { assert!(N >= 1 && N <= MAX_OUTPUT_LEN, "Blake2b output length N must be 1..=64"); } - oneshot_hash_array::(N as u8, &[], data) + let output_len = u8::try_from(N).expect("const assertion ensures the Blake2b output length fits in u8"); + oneshot_hash_array::(output_len, &[], data) } /// Compute a keyed Blake2b hash in one shot, returning a fixed-size array. @@ -1249,15 +1216,15 @@ impl Blake2b { assert!(N >= 1 && N <= MAX_OUTPUT_LEN, "Blake2b output length N must be 1..=64"); } let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); - oneshot_hash_array::(N as u8, key, data) + let output_len = u8::try_from(N).expect("const assertion ensures the Blake2b output length fits in u8"); + oneshot_hash_array::(output_len, key, data) } #[cfg(feature = "argon2")] #[inline] pub(crate) fn new_validated(output_len: usize) -> Self { debug_assert!((1..=MAX_OUTPUT_LEN).contains(&output_len)); - let output_len = output_len as u8; + let output_len = u8::try_from(output_len).expect("validated Blake2b output length fits in u8"); Self { core: Core::new(output_len, &[]), output_len, @@ -1286,10 +1253,12 @@ impl Drop for Blake2b { mod tests { use alloc::vec; - use blake2::{Blake2b as OracleBlake2b, Blake2bMac, Digest as _}; - use digest::{ - KeyInit, - consts::{U32, U64}, + use blake2::{ + Blake2b as OracleBlake2b, Blake2bMac, + digest::{ + Digest as _, Mac as _, + consts::{U32, U64}, + }, }; use super::*; @@ -1299,7 +1268,7 @@ mod tests { type OracleBlake2bMac256 = Blake2bMac; fn validated_key(key: &[u8]) -> Blake2bKey<'_> { - Blake2bKey::new(key).unwrap() + Blake2bKey::new(key).expect("test key length must satisfy the Blake2b key contract") } fn oracle_hash_256(data: &[u8]) -> [u8; 32] { @@ -1363,7 +1332,7 @@ mod tests { } #[test] - #[cfg(feature = "diag")] + #[cfg(all(feature = "diag", feature = "argon2"))] fn diag_hash_parts_portable_handles_multiblock_parts() { let first = [0x11u8; 4]; let second = [0x22u8; 1024]; @@ -1438,9 +1407,10 @@ mod tests { let key = b"secret-key"; let data = b"hello world"; - let mut oracle = OracleBlake2bMac256::new_from_slice(key).unwrap(); - hmac::Mac::update(&mut oracle, data); - let expected: [u8; 32] = hmac::Mac::finalize(oracle).into_bytes().into(); + let mut oracle = + OracleBlake2bMac256::new_from_slice(key).expect("test key length must satisfy the oracle MAC contract"); + blake2::digest::Mac::update(&mut oracle, data); + let expected: [u8; 32] = blake2::digest::Mac::finalize(oracle).into_bytes().into(); let actual = Blake2b256::keyed_digest(validated_key(key), data); assert_eq!(actual, expected); @@ -1449,9 +1419,10 @@ mod tests { #[test] fn blake2b256_keyed_empty_data() { let key = b"key"; - let mut oracle = OracleBlake2bMac256::new_from_slice(key).unwrap(); - hmac::Mac::update(&mut oracle, b""); - let expected: [u8; 32] = hmac::Mac::finalize(oracle).into_bytes().into(); + let mut oracle = + OracleBlake2bMac256::new_from_slice(key).expect("test key length must satisfy the oracle MAC contract"); + blake2::digest::Mac::update(&mut oracle, b""); + let expected: [u8; 32] = blake2::digest::Mac::finalize(oracle).into_bytes().into(); let actual = Blake2b256::keyed_digest(validated_key(key), b""); assert_eq!(actual, expected); @@ -1462,9 +1433,10 @@ mod tests { let key = &[0xAA; 64]; // max key length let data = &[0xBB; 512]; - let mut oracle = OracleBlake2bMac256::new_from_slice(key).unwrap(); - hmac::Mac::update(&mut oracle, data); - let expected: [u8; 32] = hmac::Mac::finalize(oracle).into_bytes().into(); + let mut oracle = + OracleBlake2bMac256::new_from_slice(key).expect("test key length must satisfy the oracle MAC contract"); + blake2::digest::Mac::update(&mut oracle, data); + let expected: [u8; 32] = blake2::digest::Mac::finalize(oracle).into_bytes().into(); let actual = Blake2b256::keyed_digest(validated_key(key), data); assert_eq!(actual, expected); @@ -1498,7 +1470,7 @@ mod tests { #[test] fn variable_output_1_byte() { let mut out = [0u8; 1]; - Blake2b::digest_into(b"test", &mut out).unwrap(); + Blake2b::digest_into(b"test", &mut out).expect("one-byte Blake2b output is valid"); assert_ne!(out, [0u8; 1]); } @@ -1506,13 +1478,13 @@ mod tests { fn variable_output_matches_fixed() { // 32-byte variable output should match Blake2b256 let mut var_out = [0u8; 32]; - Blake2b::digest_into(b"hello", &mut var_out).unwrap(); + Blake2b::digest_into(b"hello", &mut var_out).expect("32-byte Blake2b output is valid"); let fixed_out = Blake2b256::digest(b"hello"); assert_eq!(var_out, fixed_out); // 64-byte variable output should match Blake2b512 let mut var_out = [0u8; 64]; - Blake2b::digest_into(b"hello", &mut var_out).unwrap(); + Blake2b::digest_into(b"hello", &mut var_out).expect("64-byte Blake2b output is valid"); let fixed_out = Blake2b512::digest(b"hello"); assert_eq!(var_out, fixed_out); } @@ -1526,7 +1498,7 @@ mod tests { let expected_arr: [u8; $nn] = oracle.finalize().into(); let mut actual = [0u8; $nn]; - Blake2b::digest_into(data, &mut actual).unwrap(); + Blake2b::digest_into(data, &mut actual).expect("oracle output length is valid for Blake2b"); assert_eq!(actual, expected_arr, "Blake2b nn={} oracle mismatch", $nn); }}; } @@ -1553,7 +1525,7 @@ mod tests { fn variable_output_oracle_spread_multiblock() { // Longer input (> one block) to stress the multi-block path against the oracle. use digest::consts::{U16, U32, U48, U64}; - let data: [u8; 300] = core::array::from_fn(|i| (i & 0xff) as u8); + let data: [u8; 300] = core::array::from_fn(|i| i.to_le_bytes()[0]); assert_blake2b_var_matches!(16, U16, &data); assert_blake2b_var_matches!(32, U32, &data); assert_blake2b_var_matches!(48, U48, &data); @@ -1563,23 +1535,25 @@ mod tests { #[test] fn variable_output_all_lengths_self_consistent() { // 1..=64 oneshot == streaming(single update) == streaming(chunked) - let data: [u8; 200] = core::array::from_fn(|i| ((i * 31 + 17) & 0xff) as u8); + let data: [u8; 200] = core::array::from_fn(|i| i.strict_mul(31).strict_add(17).to_le_bytes()[0]); for nn in 1usize..=64 { let mut oneshot = vec![0u8; nn]; - Blake2b::digest_into(&data, &mut oneshot).unwrap(); + Blake2b::digest_into(&data, &mut oneshot).expect("loop output length is valid for Blake2b"); - let mut h = Blake2b::new(nn).unwrap(); + let mut h = Blake2b::new(nn).expect("loop output length is valid for Blake2b"); h.update(&data); let mut single_update = vec![0u8; nn]; - h.finalize_into(&mut single_update).unwrap(); + h.finalize_into(&mut single_update) + .expect("final output length matches the configured Blake2b length"); assert_eq!(oneshot, single_update, "single-update nn={nn}"); - let mut h = Blake2b::new(nn).unwrap(); + let mut h = Blake2b::new(nn).expect("loop output length is valid for Blake2b"); for chunk in data.chunks(37) { h.update(chunk); } let mut chunked = vec![0u8; nn]; - h.finalize_into(&mut chunked).unwrap(); + h.finalize_into(&mut chunked) + .expect("final output length matches the configured Blake2b length"); assert_eq!(oneshot, chunked, "chunked nn={nn}"); } } @@ -1589,14 +1563,15 @@ mod tests { let data = [0x5Au8; 300]; for nn in [1usize, 16, 32, 48, 63, 64] { let mut expected = vec![0u8; nn]; - Blake2b::digest_into(&data, &mut expected).unwrap(); + Blake2b::digest_into(&data, &mut expected).expect("case output length is valid for Blake2b"); - let mut h = Blake2b::new(nn).unwrap(); + let mut h = Blake2b::new(nn).expect("case output length is valid for Blake2b"); for chunk in data.chunks(37) { h.update(chunk); } let mut actual = vec![0u8; nn]; - h.finalize_into(&mut actual).unwrap(); + h.finalize_into(&mut actual) + .expect("final output length matches the configured Blake2b length"); assert_eq!(actual, expected, "streaming mismatch nn={nn}"); } } @@ -1609,11 +1584,11 @@ mod tests { let mut actual_32 = [0u8; 32]; let key = validated_key(key); - Blake2b::keyed_digest_into(key, data, &mut actual_32).unwrap(); + Blake2b::keyed_digest_into(key, data, &mut actual_32).expect("32-byte keyed output is valid"); assert_eq!(actual_32, Blake2b256::keyed_digest(key, data)); let mut actual_64 = [0u8; 64]; - Blake2b::keyed_digest_into(key, data, &mut actual_64).unwrap(); + Blake2b::keyed_digest_into(key, data, &mut actual_64).expect("64-byte keyed output is valid"); assert_eq!(actual_64, Blake2b512::keyed_digest(key, data)); } @@ -1623,10 +1598,11 @@ mod tests { let data = b"message"; for nn in [1usize, 16, 40, 48, 63] { let mut keyed = vec![0u8; nn]; - Blake2b::keyed_digest_into(validated_key(key), data, &mut keyed).unwrap(); + Blake2b::keyed_digest_into(validated_key(key), data, &mut keyed) + .expect("case output length is valid for keyed Blake2b"); let mut unkeyed = vec![0u8; nn]; - Blake2b::digest_into(data, &mut unkeyed).unwrap(); + Blake2b::digest_into(data, &mut unkeyed).expect("case output length is valid for Blake2b"); assert_ne!(keyed, unkeyed, "keyed vs unkeyed nn={nn}"); } @@ -1640,26 +1616,28 @@ mod tests { let mut a = vec![0u8; nn]; let mut b = vec![0u8; nn]; let key = validated_key(key); - Blake2b::keyed_digest_into(key, data, &mut a).unwrap(); - Blake2b::keyed_digest_into(key, data, &mut b).unwrap(); + Blake2b::keyed_digest_into(key, data, &mut a).expect("case output length is valid for keyed Blake2b"); + Blake2b::keyed_digest_into(key, data, &mut b).expect("case output length is valid for keyed Blake2b"); assert_eq!(a, b, "determinism nn={nn}"); } } #[test] fn variable_output_reset_preserves_length() { - let mut h = Blake2b::new(40).unwrap(); + let mut h = Blake2b::new(40).expect("40-byte Blake2b output is valid"); h.update(b"first"); let mut first = [0u8; 40]; - h.finalize_into(&mut first).unwrap(); + h.finalize_into(&mut first) + .expect("final output length matches the configured Blake2b length"); h.reset(); h.update(b"second"); let mut second = [0u8; 40]; - h.finalize_into(&mut second).unwrap(); + h.finalize_into(&mut second) + .expect("final output length matches the configured Blake2b length"); let mut expected = [0u8; 40]; - Blake2b::digest_into(b"second", &mut expected).unwrap(); + Blake2b::digest_into(b"second", &mut expected).expect("40-byte Blake2b output is valid"); assert_eq!(second, expected); assert_ne!(first, second); } @@ -1668,7 +1646,7 @@ mod tests { fn variable_output_digest_array_matches_into() { let arr = Blake2b::digest_array::<20>(b"hello world"); let mut expected = [0u8; 20]; - Blake2b::digest_into(b"hello world", &mut expected).unwrap(); + Blake2b::digest_into(b"hello world", &mut expected).expect("20-byte Blake2b output is valid"); assert_eq!(arr, expected); } @@ -1678,13 +1656,16 @@ mod tests { .salt(*b"domain-salt\0\0\0\0\0") .personal(*b"app-v1\0\0\0\0\0\0\0\0\0\0"); let mut oneshot = [0u8; 24]; - params.hash_into(b"hello world", &mut oneshot).unwrap(); + params + .hash_into(b"hello world", &mut oneshot) + .expect("24-byte parameterized Blake2b output is valid"); - let mut h = params.build(24).unwrap(); + let mut h = params.build(24).expect("24-byte Blake2b output is valid"); h.update(b"hello "); h.update(b"world"); let mut streamed = [0u8; 24]; - h.finalize_into(&mut streamed).unwrap(); + h.finalize_into(&mut streamed) + .expect("final output length matches the configured Blake2b length"); assert_eq!(oneshot, streamed); } @@ -1693,10 +1674,12 @@ mod tests { // Empty key/salt/personal + variable length should match direct Blake2b. for nn in [1usize, 17, 32, 48, 64] { let mut via_params = vec![0u8; nn]; - Blake2bParams::new().hash_into(b"msg", &mut via_params).unwrap(); + Blake2bParams::new() + .hash_into(b"msg", &mut via_params) + .expect("case output length is valid for parameterized Blake2b"); let mut direct = vec![0u8; nn]; - Blake2b::digest_into(b"msg", &mut direct).unwrap(); + Blake2b::digest_into(b"msg", &mut direct).expect("case output length is valid for Blake2b"); assert_eq!(via_params, direct); } } @@ -1712,12 +1695,18 @@ mod tests { #[test] fn variable_output_zero_is_rejected() { - assert_eq!(Blake2b::new(0).unwrap_err(), Blake2Error::InvalidOutputLength); + assert_eq!( + Blake2b::new(0).expect_err("zero-length Blake2b output must be rejected"), + Blake2Error::InvalidOutputLength + ); } #[test] fn variable_output_over_64_is_rejected() { - assert_eq!(Blake2b::new(65).unwrap_err(), Blake2Error::InvalidOutputLength); + assert_eq!( + Blake2b::new(65).expect_err("oversized Blake2b output must be rejected"), + Blake2Error::InvalidOutputLength + ); } #[test] @@ -1730,7 +1719,7 @@ mod tests { #[test] fn variable_output_finalize_wrong_len_is_rejected() { - let h = Blake2b::new(32).unwrap(); + let h = Blake2b::new(32).expect("32-byte Blake2b output is valid"); let mut out = [0u8; 16]; assert_eq!(h.finalize_into(&mut out), Err(Blake2Error::OutputLengthMismatch)); } @@ -1766,12 +1755,18 @@ mod tests { #[test] fn keyed_empty_key_is_rejected() { - assert_eq!(Blake2bKey::new(b"").unwrap_err(), Blake2Error::InvalidKeyLength); + assert_eq!( + Blake2bKey::new(b"").expect_err("empty Blake2b keys must be rejected"), + Blake2Error::InvalidKeyLength + ); } #[test] fn keyed_overlength_key_is_rejected() { - assert_eq!(Blake2bKey::new(&[0u8; 65]).unwrap_err(), Blake2Error::InvalidKeyLength); + assert_eq!( + Blake2bKey::new(&[0u8; 65]).expect_err("oversized Blake2b keys must be rejected"), + Blake2Error::InvalidKeyLength + ); } // ── Finalize is non-destructive ─────────────────────────────────────── @@ -1893,7 +1888,7 @@ mod tests { .personal(*b"personal\0\0\0\0\0\0\0\0"); let mut h = params.build_256(); h.update(b"first"); - let _ = h.finalize(); + let _first = h.finalize(); h.reset(); h.update(b"hello world"); @@ -1955,9 +1950,10 @@ mod tests { // Keyed Blake2b-256 for &key in &[&b"key"[..], &[0xAA; 64]] { - let mut oracle = OracleBlake2bMac256::new_from_slice(key).unwrap(); - hmac::Mac::update(&mut oracle, b"message"); - let expected: [u8; 32] = hmac::Mac::finalize(oracle).into_bytes().into(); + let mut oracle = + OracleBlake2bMac256::new_from_slice(key).expect("test key length must satisfy the oracle MAC contract"); + blake2::digest::Mac::update(&mut oracle, b"message"); + let expected: [u8; 32] = blake2::digest::Mac::finalize(oracle).into_bytes().into(); let mut h = Blake2b256::keyed_with_compress_for_test(key, compress, compress_blocks); h.update(b"message"); diff --git a/src/hashes/crypto/blake2b/riscv64.rs b/src/hashes/crypto/blake2b/riscv64.rs index 4964e191..74bec92a 100644 --- a/src/hashes/crypto/blake2b/riscv64.rs +++ b/src/hashes/crypto/blake2b/riscv64.rs @@ -15,9 +15,6 @@ //! //! Requires the V extension. Caller must verify `riscv::V`. -#![allow(unsafe_code)] -#![allow(clippy::cast_possible_truncation, clippy::indexing_slicing)] - use super::kernels::{SIGMA, init_v, load_msg}; // ─── Inline asm helpers ─────────────────────────────────────────────────── @@ -70,47 +67,26 @@ fn vxor(a: [u64; 2], b: [u64; 2]) -> [u64; 2] { // ─── G function on register pairs ───────────────────────────────────────── +struct PairState { + a: [[u64; 2]; 2], + b: [[u64; 2]; 2], + c: [[u64; 2]; 2], + d: [[u64; 2]; 2], +} + /// Blake2b G mixing on 2-wide pairs. #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn g2( - a0: &mut [u64; 2], - a1: &mut [u64; 2], - b0: &mut [u64; 2], - b1: &mut [u64; 2], - c0: &mut [u64; 2], - c1: &mut [u64; 2], - d0: &mut [u64; 2], - d1: &mut [u64; 2], - mx0: [u64; 2], - mx1: [u64; 2], - my0: [u64; 2], - my1: [u64; 2], -) { - // a += b + mx - *a0 = vadd(vadd(*a0, *b0), mx0); - *a1 = vadd(vadd(*a1, *b1), mx1); - // d = (d ^ a) >>> 32 - *d0 = ror32(vxor(*d0, *a0)); - *d1 = ror32(vxor(*d1, *a1)); - // c += d - *c0 = vadd(*c0, *d0); - *c1 = vadd(*c1, *d1); - // b = (b ^ c) >>> 24 - *b0 = ror24(vxor(*b0, *c0)); - *b1 = ror24(vxor(*b1, *c1)); - // a += b + my - *a0 = vadd(vadd(*a0, *b0), my0); - *a1 = vadd(vadd(*a1, *b1), my1); - // d = (d ^ a) >>> 16 - *d0 = ror16(vxor(*d0, *a0)); - *d1 = ror16(vxor(*d1, *a1)); - // c += d - *c0 = vadd(*c0, *d0); - *c1 = vadd(*c1, *d1); - // b = (b ^ c) >>> 63 - *b0 = ror63(vxor(*b0, *c0)); - *b1 = ror63(vxor(*b1, *c1)); +fn g2(state: &mut PairState, mx: [[u64; 2]; 2], my: [[u64; 2]; 2]) { + for pair in 0..2 { + state.a[pair] = vadd(vadd(state.a[pair], state.b[pair]), mx[pair]); + state.d[pair] = ror32(vxor(state.d[pair], state.a[pair])); + state.c[pair] = vadd(state.c[pair], state.d[pair]); + state.b[pair] = ror24(vxor(state.b[pair], state.c[pair])); + state.a[pair] = vadd(vadd(state.a[pair], state.b[pair]), my[pair]); + state.d[pair] = ror16(vxor(state.d[pair], state.a[pair])); + state.c[pair] = vadd(state.c[pair], state.d[pair]); + state.b[pair] = ror63(vxor(state.b[pair], state.c[pair])); + } } // ─── Diagonalize / Un-diagonalize ───────────────────────────────────────── @@ -121,54 +97,32 @@ fn g2( /// new_lo = [old_lo[1], old_hi[0]] /// new_hi = [old_hi[1], old_lo[0]] #[inline(always)] -fn diagonalize( - b0: &mut [u64; 2], - b1: &mut [u64; 2], - c0: &mut [u64; 2], - c1: &mut [u64; 2], - d0: &mut [u64; 2], - d1: &mut [u64; 2], -) { +fn diagonalize(state: &mut PairState) { // B: rotate left 1 - let tb0 = *b0; - let tb1 = *b1; - *b0 = [tb0[1], tb1[0]]; - *b1 = [tb1[1], tb0[0]]; + let [b0, b1] = state.b; + state.b = [[b0[1], b1[0]], [b1[1], b0[0]]]; // C: rotate left 2 = swap lo/hi - core::mem::swap(c0, c1); + state.c.swap(0, 1); // D: rotate left 3 = rotate right 1 - let td0 = *d0; - let td1 = *d1; - *d0 = [td1[1], td0[0]]; - *d1 = [td0[1], td1[0]]; + let [d0, d1] = state.d; + state.d = [[d1[1], d0[0]], [d0[1], d1[0]]]; } /// Un-diagonalize: reverse the rotations. #[inline(always)] -fn undiagonalize( - b0: &mut [u64; 2], - b1: &mut [u64; 2], - c0: &mut [u64; 2], - c1: &mut [u64; 2], - d0: &mut [u64; 2], - d1: &mut [u64; 2], -) { +fn undiagonalize(state: &mut PairState) { // B: rotate right 1 (undo left 1) - let tb0 = *b0; - let tb1 = *b1; - *b0 = [tb1[1], tb0[0]]; - *b1 = [tb0[1], tb1[0]]; + let [b0, b1] = state.b; + state.b = [[b1[1], b0[0]], [b0[1], b1[0]]]; // C: swap back - core::mem::swap(c0, c1); + state.c.swap(0, 1); // D: rotate left 1 (undo right 1) - let td0 = *d0; - let td1 = *d1; - *d0 = [td0[1], td1[0]]; - *d1 = [td1[1], td0[0]]; + let [d0, d1] = state.d; + state.d = [[d0[1], d1[0]], [d1[1], d0[0]]]; } // ─── Compress entry point ───────────────────────────────────────────────── @@ -188,14 +142,12 @@ pub(super) unsafe fn compress_rvv(h: &mut [u64; 8], block: &[u8; 128], t: u128, let v = init_v(h, t, last); // Pack into 2-wide pairs: (lo, hi) for each row - let mut a0 = [v[0], v[1]]; - let mut a1 = [v[2], v[3]]; - let mut b0 = [v[4], v[5]]; - let mut b1 = [v[6], v[7]]; - let mut c0 = [v[8], v[9]]; - let mut c1 = [v[10], v[11]]; - let mut d0 = [v[12], v[13]]; - let mut d1 = [v[14], v[15]]; + let mut state = PairState { + a: [[v[0], v[1]], [v[2], v[3]]], + b: [[v[4], v[5]], [v[6], v[7]]], + c: [[v[8], v[9]], [v[10], v[11]]], + d: [[v[12], v[13]], [v[14], v[15]]], + }; // 12 rounds for round in 0..12u8 { @@ -207,11 +159,9 @@ pub(super) unsafe fn compress_rvv(h: &mut [u64; 8], block: &[u8; 128], t: u128, let my0 = [m[s[1] as usize], m[s[3] as usize]]; let my1 = [m[s[5] as usize], m[s[7] as usize]]; - g2( - &mut a0, &mut a1, &mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1, mx0, mx1, my0, my1, - ); + g2(&mut state, [mx0, mx1], [my0, my1]); - diagonalize(&mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1); + diagonalize(&mut state); // Diagonal step let mx0 = [m[s[8] as usize], m[s[10] as usize]]; @@ -219,20 +169,18 @@ pub(super) unsafe fn compress_rvv(h: &mut [u64; 8], block: &[u8; 128], t: u128, let my0 = [m[s[9] as usize], m[s[11] as usize]]; let my1 = [m[s[13] as usize], m[s[15] as usize]]; - g2( - &mut a0, &mut a1, &mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1, mx0, mx1, my0, my1, - ); + g2(&mut state, [mx0, mx1], [my0, my1]); - undiagonalize(&mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1); + undiagonalize(&mut state); } // Finalize: h[i] ^= v[i] ^ v[i+8] - h[0] ^= a0[0] ^ c0[0]; - h[1] ^= a0[1] ^ c0[1]; - h[2] ^= a1[0] ^ c1[0]; - h[3] ^= a1[1] ^ c1[1]; - h[4] ^= b0[0] ^ d0[0]; - h[5] ^= b0[1] ^ d0[1]; - h[6] ^= b1[0] ^ d1[0]; - h[7] ^= b1[1] ^ d1[1]; + h[0] ^= state.a[0][0] ^ state.c[0][0]; + h[1] ^= state.a[0][1] ^ state.c[0][1]; + h[2] ^= state.a[1][0] ^ state.c[1][0]; + h[3] ^= state.a[1][1] ^ state.c[1][1]; + h[4] ^= state.b[0][0] ^ state.d[0][0]; + h[5] ^= state.b[0][1] ^ state.d[0][1]; + h[6] ^= state.b[1][0] ^ state.d[1][0]; + h[7] ^= state.b[1][1] ^ state.d[1][1]; } diff --git a/src/hashes/crypto/blake2b/wasm.rs b/src/hashes/crypto/blake2b/wasm.rs index d1c1cd1e..b0632e6b 100644 --- a/src/hashes/crypto/blake2b/wasm.rs +++ b/src/hashes/crypto/blake2b/wasm.rs @@ -11,8 +11,6 @@ //! //! Requires WASM SIMD128. Caller must verify `wasm::SIMD128`. -#![allow(clippy::cast_possible_truncation, clippy::indexing_slicing)] - #[cfg(target_arch = "wasm32")] use core::arch::wasm32::*; @@ -55,45 +53,39 @@ fn ror63(x: v128) -> v128 { /// Blake2b G mixing on SIMD rows (2-wide). #[cfg(target_arch = "wasm32")] #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn g2( - a0: &mut v128, - a1: &mut v128, - b0: &mut v128, - b1: &mut v128, - c0: &mut v128, - c1: &mut v128, - d0: &mut v128, - d1: &mut v128, - mx0: v128, - mx1: v128, - my0: v128, - my1: v128, -) { +fn g2(state: &mut WorkingState, mx: [v128; 2], my: [v128; 2]) { // a += b + mx - *a0 = i64x2_add(i64x2_add(*a0, *b0), mx0); - *a1 = i64x2_add(i64x2_add(*a1, *b1), mx1); + state.a[0] = i64x2_add(i64x2_add(state.a[0], state.b[0]), mx[0]); + state.a[1] = i64x2_add(i64x2_add(state.a[1], state.b[1]), mx[1]); // d = (d ^ a) >>> 32 - *d0 = ror32(v128_xor(*d0, *a0)); - *d1 = ror32(v128_xor(*d1, *a1)); + state.d[0] = ror32(v128_xor(state.d[0], state.a[0])); + state.d[1] = ror32(v128_xor(state.d[1], state.a[1])); // c += d - *c0 = i64x2_add(*c0, *d0); - *c1 = i64x2_add(*c1, *d1); + state.c[0] = i64x2_add(state.c[0], state.d[0]); + state.c[1] = i64x2_add(state.c[1], state.d[1]); // b = (b ^ c) >>> 24 - *b0 = ror24(v128_xor(*b0, *c0)); - *b1 = ror24(v128_xor(*b1, *c1)); + state.b[0] = ror24(v128_xor(state.b[0], state.c[0])); + state.b[1] = ror24(v128_xor(state.b[1], state.c[1])); // a += b + my - *a0 = i64x2_add(i64x2_add(*a0, *b0), my0); - *a1 = i64x2_add(i64x2_add(*a1, *b1), my1); + state.a[0] = i64x2_add(i64x2_add(state.a[0], state.b[0]), my[0]); + state.a[1] = i64x2_add(i64x2_add(state.a[1], state.b[1]), my[1]); // d = (d ^ a) >>> 16 - *d0 = ror16(v128_xor(*d0, *a0)); - *d1 = ror16(v128_xor(*d1, *a1)); + state.d[0] = ror16(v128_xor(state.d[0], state.a[0])); + state.d[1] = ror16(v128_xor(state.d[1], state.a[1])); // c += d - *c0 = i64x2_add(*c0, *d0); - *c1 = i64x2_add(*c1, *d1); + state.c[0] = i64x2_add(state.c[0], state.d[0]); + state.c[1] = i64x2_add(state.c[1], state.d[1]); // b = (b ^ c) >>> 63 - *b0 = ror63(v128_xor(*b0, *c0)); - *b1 = ror63(v128_xor(*b1, *c1)); + state.b[0] = ror63(v128_xor(state.b[0], state.c[0])); + state.b[1] = ror63(v128_xor(state.b[1], state.c[1])); +} + +#[cfg(target_arch = "wasm32")] +struct WorkingState { + a: [v128; 2], + b: [v128; 2], + c: [v128; 2], + d: [v128; 2], } // ─── Diagonalize / Un-diagonalize ───────────────────────────────────────── @@ -103,41 +95,41 @@ fn g2( /// `i64x2_shuffle` indices: 0,1 = lanes from first operand, 2,3 = from second. #[cfg(target_arch = "wasm32")] #[inline(always)] -fn diagonalize(b0: &mut v128, b1: &mut v128, c0: &mut v128, c1: &mut v128, d0: &mut v128, d1: &mut v128) { +fn diagonalize(state: &mut WorkingState) { // B: rotate left 1: (v4,v5,v6,v7) -> (v5,v6,v7,v4) - let tb0 = *b0; - let tb1 = *b1; - *b0 = i64x2_shuffle::<1, 2>(tb0, tb1); // [b0[1], b1[0]] = [v5, v6] - *b1 = i64x2_shuffle::<1, 2>(tb1, tb0); // [b1[1], b0[0]] = [v7, v4] + let tb0 = state.b[0]; + let tb1 = state.b[1]; + state.b[0] = i64x2_shuffle::<1, 2>(tb0, tb1); // [b0[1], b1[0]] = [v5, v6] + state.b[1] = i64x2_shuffle::<1, 2>(tb1, tb0); // [b1[1], b0[0]] = [v7, v4] // C: rotate left 2 = swap lo/hi - core::mem::swap(c0, c1); + state.c.swap(0, 1); // D: rotate left 3 = rotate right 1: (v12,v13,v14,v15) -> (v15,v12,v13,v14) - let td0 = *d0; - let td1 = *d1; - *d0 = i64x2_shuffle::<1, 2>(td1, td0); // [d1[1], d0[0]] = [v15, v12] - *d1 = i64x2_shuffle::<1, 2>(td0, td1); // [d0[1], d1[0]] = [v13, v14] + let td0 = state.d[0]; + let td1 = state.d[1]; + state.d[0] = i64x2_shuffle::<1, 2>(td1, td0); // [d1[1], d0[0]] = [v15, v12] + state.d[1] = i64x2_shuffle::<1, 2>(td0, td1); // [d0[1], d1[0]] = [v13, v14] } /// Un-diagonalize: reverse the rotations. #[cfg(target_arch = "wasm32")] #[inline(always)] -fn undiagonalize(b0: &mut v128, b1: &mut v128, c0: &mut v128, c1: &mut v128, d0: &mut v128, d1: &mut v128) { +fn undiagonalize(state: &mut WorkingState) { // B: rotate right 1 (undo left 1) - let tb0 = *b0; - let tb1 = *b1; - *b0 = i64x2_shuffle::<1, 2>(tb1, tb0); - *b1 = i64x2_shuffle::<1, 2>(tb0, tb1); + let tb0 = state.b[0]; + let tb1 = state.b[1]; + state.b[0] = i64x2_shuffle::<1, 2>(tb1, tb0); + state.b[1] = i64x2_shuffle::<1, 2>(tb0, tb1); // C: swap back - core::mem::swap(c0, c1); + state.c.swap(0, 1); // D: rotate left 1 (undo right 1) - let td0 = *d0; - let td1 = *d1; - *d0 = i64x2_shuffle::<1, 2>(td0, td1); - *d1 = i64x2_shuffle::<1, 2>(td1, td0); + let td0 = state.d[0]; + let td1 = state.d[1]; + state.d[0] = i64x2_shuffle::<1, 2>(td0, td1); + state.d[1] = i64x2_shuffle::<1, 2>(td1, td0); } // ─── Load helpers ───────────────────────────────────────────────────────── @@ -149,12 +141,18 @@ fn load_msg_pair(m: &[u64; 16], i0: u8, i1: u8) -> v128 { u64x2(m[i0 as usize], m[i1 as usize]) } -/// Load 2 consecutive u64 values from a pointer as v128. +/// Load two consecutive words into a SIMD vector. #[cfg(target_arch = "wasm32")] #[inline(always)] -unsafe fn vload_u64_pair(p: *const u64) -> v128 { - // SAFETY: caller ensures p is valid for 2 x u64 (16 bytes). - unsafe { v128_load(p as *const v128) } +fn load_u64_pair(words: &[u64; N], offset: usize) -> v128 { + u64x2(words[offset], words[offset.strict_add(1)]) +} + +#[cfg(target_arch = "wasm32")] +#[inline(always)] +fn store_u64_pair(words: &mut [u64; 8], offset: usize, value: v128) { + words[offset] = u64x2_extract_lane::<0>(value); + words[offset.strict_add(1)] = u64x2_extract_lane::<1>(value); } // ─── Compress entry point ───────────────────────────────────────────────── @@ -170,16 +168,13 @@ pub(super) unsafe fn compress_simd128(h: &mut [u64; 8], block: &[u8; 128], t: u1 let m = load_msg(block); let v = init_v(h, t, last); - // Pack into 2-wide SIMD rows: (lo, hi) for each row - // SAFETY: v is a [u64; 16] — pointer arithmetic is within bounds. - let mut a0 = unsafe { vload_u64_pair(v.as_ptr()) }; // v[0], v[1] - let mut a1 = unsafe { vload_u64_pair(v.as_ptr().add(2)) }; // v[2], v[3] - let mut b0 = unsafe { vload_u64_pair(v.as_ptr().add(4)) }; // v[4], v[5] - let mut b1 = unsafe { vload_u64_pair(v.as_ptr().add(6)) }; // v[6], v[7] - let mut c0 = unsafe { vload_u64_pair(v.as_ptr().add(8)) }; // v[8], v[9] - let mut c1 = unsafe { vload_u64_pair(v.as_ptr().add(10)) }; // v[10], v[11] - let mut d0 = unsafe { vload_u64_pair(v.as_ptr().add(12)) }; // v[12], v[13] - let mut d1 = unsafe { vload_u64_pair(v.as_ptr().add(14)) }; // v[14], v[15] + // Pack into 2-wide SIMD rows: (lo, hi) for each row. + let mut state = WorkingState { + a: [load_u64_pair(&v, 0), load_u64_pair(&v, 2)], + b: [load_u64_pair(&v, 4), load_u64_pair(&v, 6)], + c: [load_u64_pair(&v, 8), load_u64_pair(&v, 10)], + d: [load_u64_pair(&v, 12), load_u64_pair(&v, 14)], + }; // 12 rounds for round in 0..12u8 { @@ -191,11 +186,9 @@ pub(super) unsafe fn compress_simd128(h: &mut [u64; 8], block: &[u8; 128], t: u1 let my0 = load_msg_pair(&m, s[1], s[3]); let my1 = load_msg_pair(&m, s[5], s[7]); - g2( - &mut a0, &mut a1, &mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1, mx0, mx1, my0, my1, - ); + g2(&mut state, [mx0, mx1], [my0, my1]); - diagonalize(&mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1); + diagonalize(&mut state); // Diagonal step let mx0 = load_msg_pair(&m, s[8], s[10]); @@ -203,30 +196,24 @@ pub(super) unsafe fn compress_simd128(h: &mut [u64; 8], block: &[u8; 128], t: u1 let my0 = load_msg_pair(&m, s[9], s[11]); let my1 = load_msg_pair(&m, s[13], s[15]); - g2( - &mut a0, &mut a1, &mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1, mx0, mx1, my0, my1, - ); + g2(&mut state, [mx0, mx1], [my0, my1]); - undiagonalize(&mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1); + undiagonalize(&mut state); } // Finalize: h[i] ^= v[i] ^ v[i+8] - // SAFETY: h is a [u64; 8] — pointer arithmetic is within bounds. - let h0 = unsafe { vload_u64_pair(h.as_ptr()) }; - let h1 = unsafe { vload_u64_pair(h.as_ptr().add(2)) }; - let h2 = unsafe { vload_u64_pair(h.as_ptr().add(4)) }; - let h3 = unsafe { vload_u64_pair(h.as_ptr().add(6)) }; - - let r0 = v128_xor(h0, v128_xor(a0, c0)); - let r1 = v128_xor(h1, v128_xor(a1, c1)); - let r2 = v128_xor(h2, v128_xor(b0, d0)); - let r3 = v128_xor(h3, v128_xor(b1, d1)); - - // SAFETY: h is a [u64; 8] — pointer arithmetic is within bounds. - unsafe { - v128_store(h.as_mut_ptr() as *mut v128, r0); - v128_store(h.as_mut_ptr().add(2) as *mut v128, r1); - v128_store(h.as_mut_ptr().add(4) as *mut v128, r2); - v128_store(h.as_mut_ptr().add(6) as *mut v128, r3); - } + let h0 = load_u64_pair(h, 0); + let h1 = load_u64_pair(h, 2); + let h2 = load_u64_pair(h, 4); + let h3 = load_u64_pair(h, 6); + + let r0 = v128_xor(h0, v128_xor(state.a[0], state.c[0])); + let r1 = v128_xor(h1, v128_xor(state.a[1], state.c[1])); + let r2 = v128_xor(h2, v128_xor(state.b[0], state.d[0])); + let r3 = v128_xor(h3, v128_xor(state.b[1], state.d[1])); + + store_u64_pair(h, 0, r0); + store_u64_pair(h, 2, r1); + store_u64_pair(h, 4, r2); + store_u64_pair(h, 6, r3); } diff --git a/src/hashes/crypto/blake2b/x86_64.rs b/src/hashes/crypto/blake2b/x86_64.rs index 30f94eb7..812854ff 100644 --- a/src/hashes/crypto/blake2b/x86_64.rs +++ b/src/hashes/crypto/blake2b/x86_64.rs @@ -12,8 +12,6 @@ //! //! `compress_avx2` requires `avx2`. `compress_avx512vl` requires `avx512f` + `avx512vl`. -#![allow(clippy::cast_possible_truncation, clippy::indexing_slicing)] - use core::arch::x86_64::*; use super::kernels::{SIGMA, init_v, load_msg}; @@ -23,6 +21,10 @@ use super::kernels::{SIGMA, init_v, load_msg}; // ─── AVX2 rotation helpers ─────────────────────────────────────────────────── /// ROR 32: swap 32-bit halves within each 64-bit lane. +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn ror32_avx2(x: __m256i) -> __m256i { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. @@ -31,17 +33,19 @@ unsafe fn ror32_avx2(x: __m256i) -> __m256i { } /// ROR 24: byte shuffle within each 128-bit lane. +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn ror24_avx2(x: __m256i) -> __m256i { - // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. // Each 64-bit lane rotates its bytes right by 3 (= rotate_right(24)). #[repr(align(32))] struct Align32([u8; 32]); static ROT24: Align32 = Align32([ 3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10, 3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10, ]); - // SAFETY: `ROT24` is 32-byte aligned by construction, so loading it with an - // aligned AVX2 load is valid and the subsequent shuffle stays in registers. + // SAFETY: the caller provides AVX2, and `ROT24` is 32-byte aligned for the aligned load. unsafe { let mask = _mm256_load_si256(ROT24.0.as_ptr().cast()); _mm256_shuffle_epi8(x, mask) @@ -49,17 +53,19 @@ unsafe fn ror24_avx2(x: __m256i) -> __m256i { } /// ROR 16: byte shuffle within each 128-bit lane. +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn ror16_avx2(x: __m256i) -> __m256i { - // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. // Each 64-bit lane rotates its bytes right by 2 (= rotate_right(16)). #[repr(align(32))] struct Align32([u8; 32]); static ROT16: Align32 = Align32([ 2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9, 2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9, ]); - // SAFETY: `ROT16` is 32-byte aligned by construction, so loading it with an - // aligned AVX2 load is valid and the subsequent shuffle stays in registers. + // SAFETY: the caller provides AVX2, and `ROT16` is 32-byte aligned for the aligned load. unsafe { let mask = _mm256_load_si256(ROT16.0.as_ptr().cast()); _mm256_shuffle_epi8(x, mask) @@ -67,6 +73,10 @@ unsafe fn ror16_avx2(x: __m256i) -> __m256i { } /// ROR 63: (x >> 63) | (x << 1) = (x + x) ^ (x >> 63). +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn ror63_avx2(x: __m256i) -> __m256i { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. @@ -76,6 +86,10 @@ unsafe fn ror63_avx2(x: __m256i) -> __m256i { // ─── AVX2 G function ───────────────────────────────────────────────────────── /// Blake2b quarter-round G on 4-wide AVX2 rows. +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn g_avx2(a: &mut __m256i, b: &mut __m256i, c: &mut __m256i, d: &mut __m256i, mx: __m256i, my: __m256i) { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. @@ -102,6 +116,10 @@ unsafe fn g_avx2(a: &mut __m256i, b: &mut __m256i, c: &mut __m256i, d: &mut __m2 // ─── Diagonalize / un-diagonalize ──────────────────────────────────────────── /// Diagonalize: rotate B left by 1, C left by 2, D left by 3. +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn diagonalize(b: &mut __m256i, c: &mut __m256i, d: &mut __m256i) { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. @@ -116,6 +134,10 @@ unsafe fn diagonalize(b: &mut __m256i, c: &mut __m256i, d: &mut __m256i) { } /// Un-diagonalize: reverse the rotations (B right 1, C swap, D left 1). +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn undiagonalize(b: &mut __m256i, c: &mut __m256i, d: &mut __m256i) { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. @@ -156,16 +178,16 @@ pub(super) unsafe fn compress_avx2(h: &mut [u64; 8], block: &[u8; 128], t: u128, // Column step: G on (0,4,8,12), (1,5,9,13), (2,6,10,14), (3,7,11,15) // _mm256_set_epi64x takes args in HIGH→LOW order: lane3, lane2, lane1, lane0 let mx = _mm256_set_epi64x( - m[s[6] as usize] as i64, - m[s[4] as usize] as i64, - m[s[2] as usize] as i64, - m[s[0] as usize] as i64, + m[s[6] as usize].cast_signed(), + m[s[4] as usize].cast_signed(), + m[s[2] as usize].cast_signed(), + m[s[0] as usize].cast_signed(), ); let my = _mm256_set_epi64x( - m[s[7] as usize] as i64, - m[s[5] as usize] as i64, - m[s[3] as usize] as i64, - m[s[1] as usize] as i64, + m[s[7] as usize].cast_signed(), + m[s[5] as usize].cast_signed(), + m[s[3] as usize].cast_signed(), + m[s[1] as usize].cast_signed(), ); g_avx2(&mut a, &mut b, &mut c, &mut d, mx, my); @@ -174,16 +196,16 @@ pub(super) unsafe fn compress_avx2(h: &mut [u64; 8], block: &[u8; 128], t: u128, // Diagonal step: G on (0,5,10,15), (1,6,11,12), (2,7,8,13), (3,4,9,14) let mx = _mm256_set_epi64x( - m[s[14] as usize] as i64, - m[s[12] as usize] as i64, - m[s[10] as usize] as i64, - m[s[8] as usize] as i64, + m[s[14] as usize].cast_signed(), + m[s[12] as usize].cast_signed(), + m[s[10] as usize].cast_signed(), + m[s[8] as usize].cast_signed(), ); let my = _mm256_set_epi64x( - m[s[15] as usize] as i64, - m[s[13] as usize] as i64, - m[s[11] as usize] as i64, - m[s[9] as usize] as i64, + m[s[15] as usize].cast_signed(), + m[s[13] as usize].cast_signed(), + m[s[11] as usize].cast_signed(), + m[s[9] as usize].cast_signed(), ); g_avx2(&mut a, &mut b, &mut c, &mut d, mx, my); @@ -212,6 +234,10 @@ pub(super) unsafe fn compress_avx2(h: &mut [u64; 8], block: &[u8; 128], t: u128, /// Blake2b quarter-round G on 4-wide AVX-512VL rows. /// /// All rotations use `VPRORQ` (`_mm256_ror_epi64`) — no shuffle tables needed. +/// +/// # Safety +/// +/// Caller must ensure AVX-512F and AVX-512VL are available. #[inline(always)] unsafe fn g_avx512vl(a: &mut __m256i, b: &mut __m256i, c: &mut __m256i, d: &mut __m256i, mx: __m256i, my: __m256i) { // SAFETY: AVX-512VL intrinsics are available via the caller's #[target_feature] attribute. @@ -265,16 +291,16 @@ pub(super) unsafe fn compress_avx512vl(h: &mut [u64; 8], block: &[u8; 128], t: u // Column step let mx = _mm256_set_epi64x( - m[s[6] as usize] as i64, - m[s[4] as usize] as i64, - m[s[2] as usize] as i64, - m[s[0] as usize] as i64, + m[s[6] as usize].cast_signed(), + m[s[4] as usize].cast_signed(), + m[s[2] as usize].cast_signed(), + m[s[0] as usize].cast_signed(), ); let my = _mm256_set_epi64x( - m[s[7] as usize] as i64, - m[s[5] as usize] as i64, - m[s[3] as usize] as i64, - m[s[1] as usize] as i64, + m[s[7] as usize].cast_signed(), + m[s[5] as usize].cast_signed(), + m[s[3] as usize].cast_signed(), + m[s[1] as usize].cast_signed(), ); g_avx512vl(&mut a, &mut b, &mut c, &mut d, mx, my); @@ -283,16 +309,16 @@ pub(super) unsafe fn compress_avx512vl(h: &mut [u64; 8], block: &[u8; 128], t: u // Diagonal step let mx = _mm256_set_epi64x( - m[s[14] as usize] as i64, - m[s[12] as usize] as i64, - m[s[10] as usize] as i64, - m[s[8] as usize] as i64, + m[s[14] as usize].cast_signed(), + m[s[12] as usize].cast_signed(), + m[s[10] as usize].cast_signed(), + m[s[8] as usize].cast_signed(), ); let my = _mm256_set_epi64x( - m[s[15] as usize] as i64, - m[s[13] as usize] as i64, - m[s[11] as usize] as i64, - m[s[9] as usize] as i64, + m[s[15] as usize].cast_signed(), + m[s[13] as usize].cast_signed(), + m[s[11] as usize].cast_signed(), + m[s[9] as usize].cast_signed(), ); g_avx512vl(&mut a, &mut b, &mut c, &mut d, mx, my); diff --git a/src/hashes/crypto/blake2s/kernels.rs b/src/hashes/crypto/blake2s/kernels.rs index 73f6c76e..5ea6add5 100644 --- a/src/hashes/crypto/blake2s/kernels.rs +++ b/src/hashes/crypto/blake2s/kernels.rs @@ -1,5 +1,6 @@ //! Blake2s portable compression function and kernel dispatch (RFC 7693). +#[cfg(any(test, not(all(target_arch = "aarch64", target_os = "macos"))))] use crate::platform::Caps; #[cfg(target_arch = "riscv64")] use crate::platform::caps::riscv; @@ -18,8 +19,7 @@ pub(crate) type CompressBlocksFn = fn(&mut [u32; 8], &[u8], &mut u64); #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -#[cfg_attr(target_os = "macos", allow(dead_code))] -pub enum Blake2sKernelId { +pub(crate) enum Blake2sKernelId { Portable = 0, #[cfg(target_arch = "x86_64")] X86Avx2 = 1, @@ -35,7 +35,7 @@ impl Blake2sKernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "x86_64")] @@ -77,14 +77,12 @@ fn compress_x86_avx512vl(h: &mut [u32; 8], block: &[u8; 64], t: u64, last: bool) #[inline(always)] fn compress_blocks_with(h: &mut [u32; 8], blocks: &[u8], t: &mut u64, compress: CompressFn) { debug_assert_eq!(blocks.len() % 64, 0); - let mut chunks = blocks.chunks_exact(64); - for chunk in &mut chunks { + let (chunks, remainder) = blocks.as_chunks::<64>(); + for block in chunks { *t = t.strict_add(64); - // SAFETY: `chunks_exact(64)` yields slices of exactly 64 bytes. - let block = unsafe { &*chunk.as_ptr().cast::<[u8; 64]>() }; compress(h, block, *t, false); } - debug_assert!(chunks.remainder().is_empty()); + debug_assert!(remainder.is_empty()); } fn compress_blocks_portable(h: &mut [u32; 8], blocks: &[u8], t: &mut u64) { @@ -143,10 +141,10 @@ pub(crate) fn compress_blocks_fn(id: Blake2sKernelId) -> CompressBlocksFn { } /// Capabilities required to run the given kernel. +#[cfg(any(test, not(all(target_arch = "aarch64", target_os = "macos"))))] #[inline] #[must_use] -#[allow(dead_code)] // Used by runtime dispatch on targets that don't bypass to a fixed kernel. -pub const fn required_caps(id: Blake2sKernelId) -> Caps { +pub(crate) const fn required_caps(id: Blake2sKernelId) -> Caps { match id { Blake2sKernelId::Portable => Caps::NONE, #[cfg(target_arch = "x86_64")] @@ -162,7 +160,7 @@ pub const fn required_caps(id: Blake2sKernelId) -> Caps { /// All kernel IDs for agreement testing. #[cfg(test)] -pub const ALL: &[Blake2sKernelId] = &[ +pub(crate) const ALL: &[Blake2sKernelId] = &[ Blake2sKernelId::Portable, #[cfg(target_arch = "x86_64")] Blake2sKernelId::X86Avx2, @@ -197,7 +195,6 @@ pub(crate) fn compile_time_best() -> CompressFn { { return compress_x86_avx2; } - #[allow(unreachable_code)] compress } @@ -215,7 +212,6 @@ pub(crate) fn compile_time_best_blocks() -> CompressBlocksFn { { return compress_blocks_x86_avx2; } - #[allow(unreachable_code)] compress_blocks_portable } @@ -232,7 +228,6 @@ pub(crate) const IV: [u32; 8] = [ ]; /// Message-word permutation schedule (10 rows, reused cyclically for 10 rounds). -#[allow(dead_code)] // Used by target-specific SIMD backends that are not compiled on every host. pub(crate) const SIGMA: [[u8; 16]; 10] = [ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3], @@ -386,7 +381,6 @@ fn round(v: &mut [U32x4; 4], m: &[u32; 16], s: &[u8; 16]) { /// Load 16 little-endian u32 message words from a 64-byte block. #[inline(always)] -#[allow(clippy::indexing_slicing)] pub(crate) fn load_msg(block: &[u8; 64]) -> [u32; 16] { let mut m = [0u32; 16]; let src = block.as_ptr(); @@ -398,18 +392,28 @@ pub(crate) fn load_msg(block: &[u8; 64]) -> [u32; 16] { m } +#[inline(always)] +fn split_counter(counter: u64) -> (u32, u32) { + let [b0, b1, b2, b3, b4, b5, b6, b7] = counter.to_le_bytes(); + ( + u32::from_le_bytes([b0, b1, b2, b3]), + u32::from_le_bytes([b4, b5, b6, b7]), + ) +} + /// Initialize the 16-word working vector. #[cfg(any(target_arch = "x86_64", target_arch = "wasm32", target_arch = "riscv64"))] #[inline(always)] pub(crate) fn init_v(h: &[u32; 8], t: u64, last: bool) -> [u32; 16] { + let (t0, t1) = split_counter(t); let mut v = [0u32; 16]; v[..8].copy_from_slice(h); v[8] = IV[0]; v[9] = IV[1]; v[10] = IV[2]; v[11] = IV[3]; - v[12] = IV[4] ^ (t as u32); - v[13] = IV[5] ^ ((t >> 32) as u32); + v[12] = IV[4] ^ t0; + v[13] = IV[5] ^ t1; v[14] = if last { IV[6] ^ u32::MAX } else { IV[6] }; v[15] = IV[7]; v @@ -419,11 +423,9 @@ pub(crate) fn init_v(h: &[u32; 8], t: u64, last: bool) -> [u32; 16] { /// /// `t` is the total number of input bytes after this block (inclusive). /// `last` is `true` for the final block (sets the finalization flag). -#[allow(clippy::indexing_slicing)] pub(crate) fn compress(h: &mut [u32; 8], block: &[u8; 64], t: u64, last: bool) { let m = load_msg(block); - let t0 = t as u32; - let t1 = (t >> 32) as u32; + let (t0, t1) = split_counter(t); let f0 = if last { u32::MAX } else { 0 }; let mut v = [ diff --git a/src/hashes/crypto/blake2s/mod.rs b/src/hashes/crypto/blake2s/mod.rs index 010f16b4..3b7dbfeb 100644 --- a/src/hashes/crypto/blake2s/mod.rs +++ b/src/hashes/crypto/blake2s/mod.rs @@ -47,12 +47,13 @@ use kernels::IV; use super::Blake2Error; use crate::traits::{Digest, ct}; -const BLOCK_SIZE: usize = 64; -const MAX_KEY_LEN: usize = 32; -const MAX_OUTPUT_LEN: usize = 32; +const BLOCK_SIZE_U8: u8 = 64; +const BLOCK_SIZE: usize = BLOCK_SIZE_U8 as usize; +const MAX_KEY_LEN_U8: u8 = 32; +const MAX_KEY_LEN: usize = MAX_KEY_LEN_U8 as usize; +const MAX_OUTPUT_LEN_U8: u8 = 32; -#[cfg(any(test, feature = "diag"))] -#[allow(dead_code)] +#[cfg(feature = "diag")] #[inline] #[must_use] pub(crate) fn kernel_name_for_len(len: usize) -> &'static str { @@ -155,15 +156,14 @@ impl Core { Self::new_with_params(nn, key, &[0u8; SALT_LEN], &[0u8; PERSONAL_LEN]) } - #[allow(clippy::indexing_slicing)] fn new_with_params(nn: u8, key: &[u8], salt: &[u8; SALT_LEN], personal: &[u8; PERSONAL_LEN]) -> Self { assert!( - nn >= 1 && nn as usize <= MAX_OUTPUT_LEN, + (1..=MAX_OUTPUT_LEN_U8).contains(&nn), "Blake2s output length must be 1-32" ); assert!(key.len() <= MAX_KEY_LEN, "Blake2s key must be at most 32 bytes"); - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2s key length fits in u8"); let h = init_state_with_params(nn, kk, salt, personal); let stored_key = if kk > 0 { @@ -177,7 +177,7 @@ impl Core { let mut buf = [0u8; BLOCK_SIZE]; let buf_len = if kk > 0 { buf[..key.len()].copy_from_slice(key); - BLOCK_SIZE as u8 + BLOCK_SIZE_U8 } else { 0 }; @@ -201,29 +201,23 @@ impl Core { fn zeroize_key_if_any(&mut self) { if self.kk > 0 { // SAFETY: when `kk > 0`, `self.key` was initialized in `new`. - unsafe { ct::zeroize_no_fence(&mut *self.key.as_mut_ptr()) }; + unsafe { ct::zeroize_no_fence(self.key.assume_init_mut()) }; } } #[inline(always)] fn wipe(&mut self) { - for word in self.h.iter_mut() { - // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut self.h); ct::zeroize_no_fence(&mut self.buf); self.zeroize_key_if_any(); - // SAFETY: fields are valid, aligned, dereferenceable pointers. - unsafe { - core::ptr::write_volatile(&mut self.buf_len, 0); - core::ptr::write_volatile(&mut self.t, 0); - core::ptr::write_volatile(&mut self.nn, 0); - core::ptr::write_volatile(&mut self.kk, 0); - } + ct::zeroize_no_fence(core::slice::from_mut(&mut self.buf_len)); + ct::zeroize_words_no_fence(core::slice::from_mut(&mut self.t)); + ct::zeroize_no_fence(core::slice::from_mut(&mut self.nn)); + ct::zeroize_no_fence(core::slice::from_mut(&mut self.kk)); core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } - #[cfg(any(test, feature = "diag"))] + #[cfg(test)] fn new_with_compress_for_test( nn: u8, key: &[u8], @@ -236,7 +230,6 @@ impl Core { core } - #[allow(clippy::indexing_slicing)] fn update(&mut self, data: &[u8]) { if data.is_empty() { return; @@ -248,7 +241,7 @@ impl Core { if self.buf_len > 0 && (self.buf_len as usize).strict_add(data_len) > BLOCK_SIZE { let fill = BLOCK_SIZE.strict_sub(self.buf_len as usize); self.buf[self.buf_len as usize..BLOCK_SIZE].copy_from_slice(&data[..fill]); - self.t = self.t.strict_add(BLOCK_SIZE as u64); + self.t = self.t.strict_add(u64::from(BLOCK_SIZE_U8)); (self.compress)(&mut self.h, &self.buf, self.t, false); self.buf_len = 0; offset = fill; @@ -256,7 +249,7 @@ impl Core { let available = data_len.strict_sub(offset); if available > BLOCK_SIZE { - let blocks_len = available.strict_sub(1) / BLOCK_SIZE * BLOCK_SIZE; + let blocks_len = (available.strict_sub(1) / BLOCK_SIZE).strict_mul(BLOCK_SIZE); let end = offset.strict_add(blocks_len); (self.compress_blocks)(&mut self.h, &data[offset..end], &mut self.t); offset = end; @@ -266,11 +259,11 @@ impl Core { if remaining > 0 { let start = self.buf_len as usize; self.buf[start..start.strict_add(remaining)].copy_from_slice(&data[offset..]); - self.buf_len = self.buf_len.strict_add(remaining as u8); + let remaining = u8::try_from(remaining).expect("Blake2s buffer remainder fits in u8"); + self.buf_len = self.buf_len.strict_add(remaining); } } - #[allow(clippy::indexing_slicing)] fn finalize_into(&self, out: &mut [u8]) { debug_assert!(out.len() == self.nn as usize); @@ -287,30 +280,21 @@ impl Core { write_output(&h, self.nn, out); - for word in h.iter_mut() { - // SAFETY: Volatile zeroization of a stack word is sound because: - // 1. `word` comes from `h.iter_mut()`, so it is valid and uniquely borrowed. - // 2. `u32` has no invalid bit patterns; writing zero preserves validity. - // 3. The pointer does not escape this loop iteration. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut h); } /// Reset to the initial state (including re-buffering the key if keyed and /// re-applying any salt/personalization parameter-block bytes). - #[allow(clippy::indexing_slicing)] fn reset(&mut self) { self.h = init_state_with_params(self.nn, self.kk, &self.salt, &self.personal); if self.kk > 0 { let key_len = self.kk as usize; self.buf = [0u8; BLOCK_SIZE]; - // SAFETY: when `kk > 0`, `self.key` was initialized in `new`, and `self.buf` - // has at least `key_len` bytes available. - unsafe { - core::ptr::copy_nonoverlapping(self.key.as_ptr().cast::(), self.buf.as_mut_ptr(), key_len); - } - self.buf_len = BLOCK_SIZE as u8; + // SAFETY: when `kk > 0`, `self.key` was initialized in `new`. + let key = unsafe { self.key.assume_init_ref() }; + self.buf[..key_len].copy_from_slice(&key[..key_len]); + self.buf_len = BLOCK_SIZE_U8; } else { self.buf = [0u8; BLOCK_SIZE]; self.buf_len = 0; @@ -346,7 +330,6 @@ const BLAKE2S256_H0: [u32; 8] = init_unkeyed_fixed_state(32); /// digest length / key length / fanout / depth bits, and `h[4..8]` carry the /// salt and personalization words. #[inline] -#[allow(clippy::indexing_slicing)] fn init_state_with_params(nn: u8, kk: u8, salt: &[u8; SALT_LEN], personal: &[u8; PERSONAL_LEN]) -> [u32; 8] { let p0 = nn as u32 | ((kk as u32) << 8) | (1u32 << 16) | (1u32 << 24); let mut h = IV; @@ -364,7 +347,6 @@ fn init_state_with_params(nn: u8, kk: u8, salt: &[u8; SALT_LEN], personal: &[u8; } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_output(h: &[u32; 8], nn: u8, out: &mut [u8]) { let nn = nn as usize; let full_words = nn / 4; @@ -393,7 +375,6 @@ fn write_output(h: &[u32; 8], nn: u8, out: &mut [u8]) { } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn oneshot_small_into_with_params( nn: u8, key: &[u8], @@ -402,7 +383,7 @@ fn oneshot_small_into_with_params( out: &mut [u8], compress: kernels::CompressFn, ) { - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2s key length fits in u8"); let mut h = match params { Some((salt, personal)) => init_state_with_params(nn, kk, salt, personal), None => init_state(nn, kk), @@ -420,10 +401,10 @@ fn oneshot_small_into_with_params( key_block[..key.len()].copy_from_slice(key); if data.is_empty() { - compress(&mut h, &key_block, BLOCK_SIZE as u64, true); + compress(&mut h, &key_block, u64::from(BLOCK_SIZE_U8), true); ct::zeroize(&mut key_block); } else { - compress(&mut h, &key_block, BLOCK_SIZE as u64, false); + compress(&mut h, &key_block, u64::from(BLOCK_SIZE_U8), false); ct::zeroize(&mut key_block); let mut data_block = [0u8; BLOCK_SIZE]; @@ -431,20 +412,16 @@ fn oneshot_small_into_with_params( compress( &mut h, &data_block, - (BLOCK_SIZE as u64).strict_add(data.len() as u64), + u64::from(BLOCK_SIZE_U8).strict_add(data.len() as u64), true, ); } write_output(&h, nn, out); - for word in &mut h { - // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut h); } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn oneshot_hash_into_inner( nn: u8, key: &[u8], @@ -454,7 +431,7 @@ fn oneshot_hash_into_inner( ) { debug_assert!(out.len() == nn as usize); assert!( - nn >= 1 && nn as usize <= MAX_OUTPUT_LEN, + (1..=MAX_OUTPUT_LEN_U8).contains(&nn), "Blake2s output length must be 1-32" ); assert!(key.len() <= MAX_KEY_LEN, "Blake2s key must be at most 32 bytes"); @@ -465,7 +442,7 @@ fn oneshot_hash_into_inner( return; } - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2s key length fits in u8"); let mut h = match params { Some((salt, personal)) => init_state_with_params(nn, kk, salt, personal), None => init_state(nn, kk), @@ -473,7 +450,7 @@ fn oneshot_hash_into_inner( let mut buf = [0u8; BLOCK_SIZE]; let mut buf_len = if kk > 0 { buf[..key.len()].copy_from_slice(key); - BLOCK_SIZE as u8 + BLOCK_SIZE_U8 } else { 0 }; @@ -486,7 +463,7 @@ fn oneshot_hash_into_inner( if fill > 0 { buf[buf_len as usize..BLOCK_SIZE].copy_from_slice(&data[..fill]); } - t = t.strict_add(BLOCK_SIZE as u64); + t = t.strict_add(u64::from(BLOCK_SIZE_U8)); compress(&mut h, &buf, t, false); ct::zeroize(&mut buf); buf_len = 0; @@ -495,7 +472,7 @@ fn oneshot_hash_into_inner( let available = data_len.strict_sub(offset); if available > BLOCK_SIZE { - let blocks_len = available.strict_sub(1) / BLOCK_SIZE * BLOCK_SIZE; + let blocks_len = (available.strict_sub(1) / BLOCK_SIZE).strict_mul(BLOCK_SIZE); let end = offset.strict_add(blocks_len); let compress_blocks = dispatch::compress_blocks_dispatch(); compress_blocks(&mut h, &data[offset..end], &mut t); @@ -505,7 +482,7 @@ fn oneshot_hash_into_inner( let remaining = data_len.strict_sub(offset); if remaining > 0 { buf[..remaining].copy_from_slice(&data[offset..]); - buf_len = remaining as u8; + buf_len = u8::try_from(remaining).expect("Blake2s buffer remainder fits in u8"); } t = t.strict_add(buf_len as u64); @@ -513,10 +490,7 @@ fn oneshot_hash_into_inner( write_output(&h, nn, out); if kk > 0 { - for word in &mut h { - // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut h); ct::zeroize(&mut buf); } } @@ -633,12 +607,11 @@ impl Blake2sParams { /// Set a validated MAC key. Omit this method for unkeyed hashing. #[must_use] - #[allow(clippy::indexing_slicing)] pub fn key(mut self, key: Blake2sKey<'_>) -> Self { let key = key.as_bytes(); self.key_buf = [0u8; MAX_KEY_LEN]; self.key_buf[..key.len()].copy_from_slice(key); - self.key_len = key.len() as u8; + self.key_len = u8::try_from(key.len()).expect("validated Blake2s key length fits in u8"); self } @@ -656,7 +629,6 @@ impl Blake2sParams { self } - #[allow(clippy::indexing_slicing)] fn key_slice(&self) -> &[u8] { &self.key_buf[..usize::from(self.key_len)] } @@ -748,7 +720,6 @@ impl Blake2s256 { #[must_use] pub fn new_keyed(key: Blake2sKey<'_>) -> Self { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2s key must not be empty"); Self(Core::new(32, key)) } @@ -756,12 +727,10 @@ impl Blake2s256 { #[must_use] pub fn keyed_digest(key: Blake2sKey<'_>, data: &[u8]) -> [u8; 32] { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2s key must not be empty"); oneshot_hash_array::<32>(32, key, data) } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn new_with_compress_for_test( compress: kernels::CompressFn, compress_blocks: kernels::CompressBlocksFn, @@ -769,8 +738,7 @@ impl Blake2s256 { Self(Core::new_with_compress_for_test(32, &[], compress, compress_blocks)) } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn keyed_with_compress_for_test( key: &[u8], compress: kernels::CompressFn, @@ -782,6 +750,7 @@ impl Blake2s256 { } #[cfg(feature = "diag")] +/// Derive a keyed Blake2s-256 diagnostic digest with the portable backend. #[must_use] pub fn diag_blake2s256_keyed_digest_portable(key: &[u8; 32]) -> [u8; 32] { let mut out = [0u8; 32]; @@ -867,7 +836,6 @@ impl Blake2s128 { #[must_use] pub fn new_keyed(key: Blake2sKey<'_>) -> Self { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2s key must not be empty"); Self(Core::new(16, key)) } @@ -875,7 +843,6 @@ impl Blake2s128 { #[must_use] pub fn keyed_digest(key: Blake2sKey<'_>, data: &[u8]) -> [u8; 16] { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2s key must not be empty"); oneshot_hash_array::<16>(16, key, data) } @@ -939,9 +906,13 @@ impl_std_io_write_for_digest!(Blake2s128); #[cfg(test)] mod tests { - use blake2::{Blake2s128 as OracleBlake2s128, Blake2s256 as OracleBlake2s256, Blake2sMac, Digest as _}; - use digest::typenum::{U16, U32}; - use hmac::{Mac as _, digest::KeyInit}; + use blake2::{ + Blake2s as OracleBlake2s, Blake2s256 as OracleBlake2s256, Blake2sMac, + digest::{ + Digest as _, Mac as _, + consts::{U16, U32}, + }, + }; use super::{ kernels::{ @@ -953,9 +924,10 @@ mod tests { type OracleBlake2sMac128 = Blake2sMac; type OracleBlake2sMac256 = Blake2sMac; + type OracleBlake2s128 = OracleBlake2s; fn validated_key(key: &[u8]) -> Blake2sKey<'_> { - Blake2sKey::new(key).unwrap() + Blake2sKey::new(key).expect("test Blake2s key length is valid") } const ORACLE_CASES: &[&[u8]] = &[ @@ -1064,7 +1036,7 @@ mod tests { let key = b"secret-key"; let data = b"hello world"; - let mut oracle = OracleBlake2sMac256::new_from_slice(key).unwrap(); + let mut oracle = OracleBlake2sMac256::new_from_slice(key).expect("oracle accepts a valid Blake2s-256 key"); oracle.update(data); let expected = oracle.finalize().into_bytes(); @@ -1077,7 +1049,7 @@ mod tests { let key = b"tiny-key"; let data = b"hello world"; - let mut oracle = OracleBlake2sMac128::new_from_slice(key).unwrap(); + let mut oracle = OracleBlake2sMac128::new_from_slice(key).expect("oracle accepts a valid Blake2s-128 key"); oracle.update(data); let expected = oracle.finalize().into_bytes(); @@ -1090,7 +1062,7 @@ mod tests { let key = &[0xAA; 32]; let data = &[0xBB; 512]; - let mut oracle = OracleBlake2sMac256::new_from_slice(key).unwrap(); + let mut oracle = OracleBlake2sMac256::new_from_slice(key).expect("oracle accepts a valid Blake2s-256 key"); oracle.update(data); let expected = oracle.finalize().into_bytes(); @@ -1134,12 +1106,18 @@ mod tests { #[test] fn keyed_empty_key_is_rejected() { - assert_eq!(Blake2sKey::new(b"").unwrap_err(), Blake2Error::InvalidKeyLength); + assert_eq!( + Blake2sKey::new(b"").expect_err("empty Blake2s keys must be rejected"), + Blake2Error::InvalidKeyLength + ); } #[test] fn keyed_overlength_key_is_rejected() { - assert_eq!(Blake2sKey::new(&[0u8; 33]).unwrap_err(), Blake2Error::InvalidKeyLength); + assert_eq!( + Blake2sKey::new(&[0u8; 33]).expect_err("overlength Blake2s keys must be rejected"), + Blake2Error::InvalidKeyLength + ); } // ── Params (salt + personalization) ─────────────────────────────────── @@ -1244,7 +1222,7 @@ mod tests { let params = Blake2sParams::new().salt(*b"salty\0\0\0").personal(*b"tagging\0"); let mut h = params.build_256(); h.update(b"first"); - let _ = h.finalize(); + let _first = h.finalize(); h.reset(); h.update(b"hello world"); @@ -1291,7 +1269,8 @@ mod tests { (&[0xAA; 16][..], &[0x55; 257][..]), (&[0xCC; 32][..], &[0x11; 512][..]), ] { - let mut oracle_128 = OracleBlake2sMac128::new_from_slice(key).unwrap(); + let mut oracle_128 = + OracleBlake2sMac128::new_from_slice(key).expect("oracle accepts the forced-kernel Blake2s-128 key"); oracle_128.update(msg); let expected_128 = oracle_128.finalize().into_bytes(); @@ -1306,7 +1285,8 @@ mod tests { key.len(), ); - let mut oracle_256 = OracleBlake2sMac256::new_from_slice(key).unwrap(); + let mut oracle_256 = + OracleBlake2sMac256::new_from_slice(key).expect("oracle accepts the forced-kernel Blake2s-256 key"); oracle_256.update(msg); let expected_256 = oracle_256.finalize().into_bytes(); diff --git a/src/hashes/crypto/blake2s/riscv64.rs b/src/hashes/crypto/blake2s/riscv64.rs index dc66ef2a..b289dc26 100644 --- a/src/hashes/crypto/blake2s/riscv64.rs +++ b/src/hashes/crypto/blake2s/riscv64.rs @@ -5,8 +5,6 @@ //! the compiler lower the hot pair/quad operations appropriately for current //! RISC-V vector hardware. -#![allow(clippy::indexing_slicing)] - use super::kernels::{SIGMA, init_v, load_msg}; #[inline(always)] diff --git a/src/hashes/crypto/blake2s/wasm.rs b/src/hashes/crypto/blake2s/wasm.rs index 1a425f05..35e69e23 100644 --- a/src/hashes/crypto/blake2s/wasm.rs +++ b/src/hashes/crypto/blake2s/wasm.rs @@ -4,8 +4,6 @@ //! `v128`, diagonalization is lane shuffling, and the 32-bit rotates map to //! byte shuffles plus shift/or. -#![allow(clippy::indexing_slicing)] - use core::arch::wasm32::*; use super::kernels::{SIGMA, init_v, load_msg}; @@ -62,9 +60,21 @@ fn load_msg_quad(m: &[u32; 16], i0: u8, i1: u8, i2: u8, i3: u8) -> v128 { } #[inline(always)] -unsafe fn vload_u32_quad(p: *const u32) -> v128 { - // SAFETY: caller ensures `p` is valid for 16 bytes / 4 u32 lanes. - unsafe { v128_load(p.cast()) } +fn load_u32_quad(words: &[u32; N], offset: usize) -> v128 { + u32x4( + words[offset], + words[offset.strict_add(1)], + words[offset.strict_add(2)], + words[offset.strict_add(3)], + ) +} + +#[inline(always)] +fn store_u32_quad(words: &mut [u32; 8], offset: usize, value: v128) { + words[offset] = u32x4_extract_lane::<0>(value); + words[offset.strict_add(1)] = u32x4_extract_lane::<1>(value); + words[offset.strict_add(2)] = u32x4_extract_lane::<2>(value); + words[offset.strict_add(3)] = u32x4_extract_lane::<3>(value); } /// Blake2s WASM SIMD128 compress. @@ -77,10 +87,10 @@ pub(super) unsafe fn compress_simd128(h: &mut [u32; 8], block: &[u8; 64], t: u64 let m = load_msg(block); let v = init_v(h, t, last); - let mut a = unsafe { vload_u32_quad(v.as_ptr()) }; - let mut b = unsafe { vload_u32_quad(v.as_ptr().add(4)) }; - let mut c = unsafe { vload_u32_quad(v.as_ptr().add(8)) }; - let mut d = unsafe { vload_u32_quad(v.as_ptr().add(12)) }; + let mut a = load_u32_quad(&v, 0); + let mut b = load_u32_quad(&v, 4); + let mut c = load_u32_quad(&v, 8); + let mut d = load_u32_quad(&v, 12); for round in 0..10u8 { let s = &SIGMA[round as usize]; @@ -98,11 +108,9 @@ pub(super) unsafe fn compress_simd128(h: &mut [u32; 8], block: &[u8; 64], t: u64 undiagonalize(&mut b, &mut c, &mut d); } - let h0 = unsafe { vload_u32_quad(h.as_ptr()) }; - let h1 = unsafe { vload_u32_quad(h.as_ptr().add(4)) }; + let h0 = load_u32_quad(h, 0); + let h1 = load_u32_quad(h, 4); - unsafe { - v128_store(h.as_mut_ptr().cast(), v128_xor(h0, v128_xor(a, c))); - v128_store(h.as_mut_ptr().add(4).cast(), v128_xor(h1, v128_xor(b, d))); - } + store_u32_quad(h, 0, v128_xor(h0, v128_xor(a, c))); + store_u32_quad(h, 4, v128_xor(h1, v128_xor(b, d))); } diff --git a/src/hashes/crypto/blake2s/x86_64.rs b/src/hashes/crypto/blake2s/x86_64.rs index 202ee7db..5445d47e 100644 --- a/src/hashes/crypto/blake2s/x86_64.rs +++ b/src/hashes/crypto/blake2s/x86_64.rs @@ -6,38 +6,36 @@ //! //! This is a real SIMD backend, not a dispatch stub. -#![allow(clippy::cast_possible_wrap, clippy::indexing_slicing)] - use core::arch::x86_64::*; use super::kernels::{SIGMA, init_v, load_msg}; #[inline(always)] -unsafe fn ror16_avx2(x: __m128i) -> __m128i { +fn ror16_avx2(x: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics operate only on the provided SIMD register. unsafe { _mm_or_si128(_mm_srli_epi32(x, 16), _mm_slli_epi32(x, 16)) } } #[inline(always)] -unsafe fn ror12_avx2(x: __m128i) -> __m128i { +fn ror12_avx2(x: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics operate only on the provided SIMD register. unsafe { _mm_or_si128(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 20)) } } #[inline(always)] -unsafe fn ror8_avx2(x: __m128i) -> __m128i { +fn ror8_avx2(x: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics operate only on the provided SIMD register. unsafe { _mm_or_si128(_mm_srli_epi32(x, 8), _mm_slli_epi32(x, 24)) } } #[inline(always)] -unsafe fn ror7_avx2(x: __m128i) -> __m128i { +fn ror7_avx2(x: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics operate only on the provided SIMD register. unsafe { _mm_or_si128(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 25)) } } #[inline(always)] -unsafe fn g_avx2(a: &mut __m128i, b: &mut __m128i, c: &mut __m128i, d: &mut __m128i, mx: __m128i, my: __m128i) { +fn g_avx2(a: &mut __m128i, b: &mut __m128i, c: &mut __m128i, d: &mut __m128i, mx: __m128i, my: __m128i) { // SAFETY: all operations stay within SIMD registers and mutate disjoint rows. unsafe { *a = _mm_add_epi32(_mm_add_epi32(*a, *b), mx); @@ -52,7 +50,7 @@ unsafe fn g_avx2(a: &mut __m128i, b: &mut __m128i, c: &mut __m128i, d: &mut __m1 } #[inline(always)] -unsafe fn diagonalize(b: &mut __m128i, c: &mut __m128i, d: &mut __m128i) { +fn diagonalize(b: &mut __m128i, c: &mut __m128i, d: &mut __m128i) { // SAFETY: shuffle permutes only the provided SIMD registers. unsafe { *b = _mm_shuffle_epi32(*b, 0x39); @@ -62,7 +60,7 @@ unsafe fn diagonalize(b: &mut __m128i, c: &mut __m128i, d: &mut __m128i) { } #[inline(always)] -unsafe fn undiagonalize(b: &mut __m128i, c: &mut __m128i, d: &mut __m128i) { +fn undiagonalize(b: &mut __m128i, c: &mut __m128i, d: &mut __m128i) { // SAFETY: shuffle permutes only the provided SIMD registers. unsafe { *b = _mm_shuffle_epi32(*b, 0x93); @@ -72,14 +70,14 @@ unsafe fn undiagonalize(b: &mut __m128i, c: &mut __m128i, d: &mut __m128i) { } #[inline(always)] -unsafe fn load_msg_quad(m: &[u32; 16], i0: u8, i1: u8, i2: u8, i3: u8) -> __m128i { +fn load_msg_quad(m: &[u32; 16], i0: u8, i1: u8, i2: u8, i3: u8) -> __m128i { // SAFETY: `_mm_set_epi32` constructs a register from in-bounds message words. unsafe { _mm_set_epi32( - m[i3 as usize] as i32, - m[i2 as usize] as i32, - m[i1 as usize] as i32, - m[i0 as usize] as i32, + m[i3 as usize].cast_signed(), + m[i2 as usize].cast_signed(), + m[i1 as usize].cast_signed(), + m[i0 as usize].cast_signed(), ) } } @@ -126,6 +124,12 @@ pub(super) unsafe fn compress_avx2(h: &mut [u32; 8], block: &[u8; 64], t: u64, l } #[inline(always)] +/// Applies one BLAKE2s mixing step with AVX-512VL rotates. +/// +/// # Safety +/// +/// The caller must execute this function only when AVX-512F and AVX-512VL are +/// enabled for the current execution scope. unsafe fn g_avx512vl(a: &mut __m128i, b: &mut __m128i, c: &mut __m128i, d: &mut __m128i, mx: __m128i, my: __m128i) { // SAFETY: all operations stay within SIMD registers and the required target // features are enabled by the caller. diff --git a/src/hashes/crypto/blake3/aarch64.rs b/src/hashes/crypto/blake3/aarch64.rs index 3364b302..028b4fec 100644 --- a/src/hashes/crypto/blake3/aarch64.rs +++ b/src/hashes/crypto/blake3/aarch64.rs @@ -12,13 +12,8 @@ //! //! # Safety //! -//! All functions in this module are marked `unsafe` and require NEON -//! to be present. Callers must verify CPU capabilities before calling. - -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::too_many_arguments)] -#![allow(clippy::many_single_char_names)] +//! Unsafe entry points require NEON to be present. Callers must verify CPU +//! capabilities before calling them. #[cfg(target_arch = "aarch64")] use core::arch::aarch64::*; @@ -32,11 +27,32 @@ mod asm; // Constants use super::{BLOCK_LEN, CHUNK_LEN, CHUNK_START, IV, MSG_SCHEDULE, OUT_LEN, PARENT, words16_from_le_bytes_64}; +const BLOCK_LEN_U32: u32 = 64; + +#[inline(always)] +fn counter_words(counter: u64) -> (u32, u32) { + let [b0, b1, b2, b3, b4, b5, b6, b7] = counter.to_le_bytes(); + ( + u32::from_le_bytes([b0, b1, b2, b3]), + u32::from_le_bytes([b4, b5, b6, b7]), + ) +} + +#[inline(always)] +fn counter_lanes(counter: u64, deltas: [u64; 4]) -> ([u32; 4], [u32; 4]) { + let [d0, d1, d2, d3] = deltas; + let w0 = counter_words(counter.wrapping_add(d0)); + let w1 = counter_words(counter.wrapping_add(d1)); + let w2 = counter_words(counter.wrapping_add(d2)); + let w3 = counter_words(counter.wrapping_add(d3)); + ([w0.0, w1.0, w2.0, w3.0], [w0.1, w1.1, w2.1, w3.1]) +} + #[cfg(any(target_os = "linux", target_os = "macos"))] #[inline(always)] fn ptr_is_aligned(ptr: *const u8, align: usize) -> bool { debug_assert!(align.is_power_of_two()); - (ptr as usize) & (align - 1) == 0 + (ptr as usize) & align.strict_sub(1) == 0 } // Strategy: @@ -69,6 +85,14 @@ struct ChunkStateAsmScratch { #[cfg(any(target_os = "linux", target_os = "macos"))] #[inline(always)] +/// Dispatches full blocks to the platform assembly compressor. +/// +/// # Safety +/// +/// The current CPU must support NEON. `blocks` must be aligned as required by +/// the assembly backend and readable for `num_blocks * BLOCK_LEN` bytes. +/// `chaining_value` must reference eight writable `u32`s, and +/// `blocks_compressed` must reference one writable byte. unsafe fn chunk_compress_blocks_asm( blocks: *const u8, chaining_value: *mut u32, @@ -112,6 +136,10 @@ static ROT8_TABLE: [u8; 16] = [1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, /// Rotate right by 16 bits (each u32 lane). /// Uses vrev32q_u16 which reverses 16-bit elements within 32-bit containers. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rotr16(v: uint32x4_t) -> uint32x4_t { @@ -125,6 +153,10 @@ unsafe fn rotr16(v: uint32x4_t) -> uint32x4_t { /// Rotate right by 12 bits (each u32 lane). /// Uses shift+OR which allows parallel execution on superscalar cores. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rotr12(v: uint32x4_t) -> uint32x4_t { @@ -138,6 +170,10 @@ unsafe fn rotr12(v: uint32x4_t) -> uint32x4_t { /// Rotate right by 8 bits (each u32 lane). /// Uses vqtbl1q_u8 (table lookup) for byte-level rotation - single cycle. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rotr8_tbl(v: uint32x4_t, tbl: uint8x16_t) -> uint32x4_t { @@ -153,6 +189,10 @@ unsafe fn rotr8_tbl(v: uint32x4_t, tbl: uint8x16_t) -> uint32x4_t { /// /// Some cores have relatively high-latency `tbl`/`vtbl` paths; for the per-block /// compressor (latency-sensitive), prefer a shift/or implementation. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rotr8(v: uint32x4_t) -> uint32x4_t { @@ -163,6 +203,10 @@ unsafe fn rotr8(v: uint32x4_t) -> uint32x4_t { /// Rotate right by 7 bits (each u32 lane). /// Uses shift+OR which allows parallel execution on superscalar cores. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rotr7(v: uint32x4_t) -> uint32x4_t { @@ -176,6 +220,10 @@ unsafe fn rotr7(v: uint32x4_t) -> uint32x4_t { // Lane rotation helpers (for diagonalization) /// Rotate lanes left by 1: [a, b, c, d] -> [b, c, d, a] +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rot_lanes_left_1(v: uint32x4_t) -> uint32x4_t { @@ -184,6 +232,10 @@ unsafe fn rot_lanes_left_1(v: uint32x4_t) -> uint32x4_t { } /// Rotate lanes left by 2: [a, b, c, d] -> [c, d, a, b] +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rot_lanes_left_2(v: uint32x4_t) -> uint32x4_t { @@ -192,6 +244,10 @@ unsafe fn rot_lanes_left_2(v: uint32x4_t) -> uint32x4_t { } /// Rotate lanes left by 3: [a, b, c, d] -> [d, a, b, c] +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rot_lanes_left_3(v: uint32x4_t) -> uint32x4_t { @@ -218,6 +274,10 @@ unsafe fn rot_lanes_left_3(v: uint32x4_t) -> uint32x4_t { /// vecs[2] = [a2, b2, c2, d2] /// vecs[3] = [a3, b3, c3, d3] /// ``` +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn transpose_vecs(vecs: &mut [uint32x4_t; 4]) { @@ -237,6 +297,11 @@ unsafe fn transpose_vecs(vecs: &mut [uint32x4_t; 4]) { /// Load and transpose message words from 4 input blocks. /// After transpose, m[i] contains word i from all 4 inputs. +/// +/// # Safety +/// +/// The current CPU must support NEON. Each input pointer must be readable for +/// `block_offset + block_len` bytes, and `block_len` must not exceed `BLOCK_LEN`. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn load_msg_vecs_transposed(inputs: [*const u8; 4], block_offset: usize, block_len: usize) -> [uint32x4_t; 16] { @@ -248,6 +313,9 @@ unsafe fn load_msg_vecs_transposed(inputs: [*const u8; 4], block_offset: usize, } #[inline(always)] + /// # Safety + /// + /// The current CPU must support NEON, and `src` must be readable for 16 bytes. unsafe fn loadu_128(src: *const u8) -> uint32x4_t { // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. @@ -265,8 +333,8 @@ unsafe fn load_msg_vecs_transposed(inputs: [*const u8; 4], block_offset: usize, let mut out = [vdupq_n_u32(0); 16]; // Load four 16-byte chunks per input and transpose each 4x4. - for lane_block in 0..4 { - let off = block_offset + lane_block * 16; + for lane_block in 0usize..4 { + let off = block_offset.strict_add(lane_block.strict_mul(16)); let mut vecs = [ loadu_128(inputs[0].add(off)), loadu_128(inputs[1].add(off)), @@ -274,10 +342,11 @@ unsafe fn load_msg_vecs_transposed(inputs: [*const u8; 4], block_offset: usize, loadu_128(inputs[3].add(off)), ]; transpose_vecs(&mut vecs); - out[lane_block * 4] = vecs[0]; - out[lane_block * 4 + 1] = vecs[1]; - out[lane_block * 4 + 2] = vecs[2]; - out[lane_block * 4 + 3] = vecs[3]; + let dst = lane_block.strict_mul(4); + out[dst] = vecs[0]; + out[dst.strict_add(1)] = vecs[1]; + out[dst.strict_add(2)] = vecs[2]; + out[dst.strict_add(3)] = vecs[3]; } return out; @@ -313,6 +382,11 @@ unsafe fn load_msg_vecs_transposed(inputs: [*const u8; 4], block_offset: usize, /// - lane 3: `base + 3 * CHUNK_LEN` /// /// This path is only valid for full 64-byte blocks. +/// +/// # Safety +/// +/// The current CPU must support NEON. `base` must be readable for four full +/// chunks, and `block_offset` must identify a complete block within each chunk. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn load_msg_vecs_transposed_contiguous(base: *const u8, block_offset: usize) -> [uint32x4_t; 16] { @@ -324,6 +398,9 @@ unsafe fn load_msg_vecs_transposed_contiguous(base: *const u8, block_offset: usi } #[inline(always)] + /// # Safety + /// + /// The current CPU must support NEON, and `src` must be readable for 16 bytes. unsafe fn loadu_128(src: *const u8) -> uint32x4_t { // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. @@ -339,18 +416,19 @@ unsafe fn load_msg_vecs_transposed_contiguous(base: *const u8, block_offset: usi let mut out = [vdupq_n_u32(0); 16]; macro_rules! load_and_transpose { ($dst:expr, $off:expr) => {{ - let off = block_offset + $off; + let off = block_offset.strict_add($off); + let dst: usize = $dst; let mut vecs = [ loadu_128(base.add(off)), - loadu_128(base.add(CHUNK_LEN + off)), - loadu_128(base.add(2 * CHUNK_LEN + off)), - loadu_128(base.add(3 * CHUNK_LEN + off)), + loadu_128(base.add(CHUNK_LEN.strict_add(off))), + loadu_128(base.add(CHUNK_LEN.strict_mul(2).strict_add(off))), + loadu_128(base.add(CHUNK_LEN.strict_mul(3).strict_add(off))), ]; transpose_vecs(&mut vecs); - out[$dst] = vecs[0]; - out[$dst + 1] = vecs[1]; - out[$dst + 2] = vecs[2]; - out[$dst + 3] = vecs[3]; + out[dst] = vecs[0]; + out[dst.strict_add(1)] = vecs[1]; + out[dst.strict_add(2)] = vecs[2]; + out[dst.strict_add(3)] = vecs[3]; }}; } @@ -365,6 +443,10 @@ unsafe fn load_msg_vecs_transposed_contiguous(base: *const u8, block_offset: usi // Single-block compression (NEON accelerated) /// Helper: take the low 64-bit lane (2x u32) from each input and concatenate. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn concat_low64_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { @@ -384,6 +466,10 @@ unsafe fn concat_low64_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { /// Each round uses the same access pattern for message words, and the message /// vectors are permuted between rounds. This avoids the expensive per-round /// gather/extract machinery that dominated the previous per-block compressor. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn permute_msg(m0: uint32x4_t, m1: uint32x4_t, m2: uint32x4_t, m3: uint32x4_t) -> [uint32x4_t; 4] { @@ -418,6 +504,12 @@ unsafe fn permute_msg(m0: uint32x4_t, m1: uint32x4_t, m2: uint32x4_t, m3: uint32 #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] #[inline] +/// Runs the NEON compression core without its final output transform. +/// +/// # Safety +/// +/// The current CPU must support NEON, and `block` must be readable for +/// `BLOCK_LEN` bytes. unsafe fn compress_neon_core( chaining_value: &[u32; 8], block: *const u8, @@ -433,6 +525,9 @@ unsafe fn compress_neon_core( } #[inline(always)] + /// # Safety + /// + /// The current CPU must support NEON, and `src` must be readable for 16 bytes. unsafe fn loadu_u32x4(src: *const u8) -> uint32x4_t { // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. @@ -452,8 +547,7 @@ unsafe fn compress_neon_core( let mut row2 = vld1q_u32(IV.as_ptr()); // Build row3 from counter, block_len, and flags. - let counter_lo = counter as u32; - let counter_hi = (counter >> 32) as u32; + let (counter_lo, counter_hi) = counter_words(counter); let row3_arr: [u32; 4] = [counter_lo, counter_hi, block_len, flags]; let mut row3 = vld1q_u32(row3_arr.as_ptr()); @@ -547,6 +641,12 @@ pub(crate) unsafe fn compress_neon( #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] +/// Compresses one byte-oriented block and returns the full compression output. +/// +/// # Safety +/// +/// The current CPU must support NEON, and `block` must be readable for +/// `BLOCK_LEN` bytes. unsafe fn compress_neon_bytes( chaining_value: &[u32; 8], block: *const u8, @@ -577,23 +677,27 @@ unsafe fn compress_neon_bytes( // 4-way parallel hashing (hash4_neon) +#[derive(Clone, Copy)] +struct Hash4Flags { + base: u32, + start: u32, + end: u32, +} + /// Parallel G function for 4 independent chunks. /// /// In this model: /// - v[0..4] are the four rows of state from 4 different chunks /// - Each vector lane i corresponds to chunk i +/// +/// # Safety +/// +/// The current CPU must support NEON. Every index in `indices` must be less +/// than 16 and the four indices must be distinct. #[cfg(target_arch = "aarch64")] #[inline(always)] -unsafe fn g4( - v: &mut [uint32x4_t; 16], - a: usize, - b: usize, - c: usize, - d: usize, - mx: uint32x4_t, - my: uint32x4_t, - rot8_tbl: uint8x16_t, -) { +unsafe fn g4(v: &mut [uint32x4_t; 16], indices: [usize; 4], mx: uint32x4_t, my: uint32x4_t, rot8_tbl: uint8x16_t) { + let [a, b, c, d] = indices; // SAFETY: NEON intrinsics are available via this function's #[target_feature] attribute. unsafe { // a = a + b + mx @@ -622,6 +726,11 @@ unsafe fn g4( } /// One round of the parallel compression function for 4 chunks. +/// +/// # Safety +/// +/// The current CPU must support NEON, and `r` must be less than +/// `MSG_SCHEDULE.len()`. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn round4(v: &mut [uint32x4_t; 16], m: &[uint32x4_t; 16], r: usize, rot8_tbl: uint8x16_t) { @@ -635,24 +744,30 @@ unsafe fn round4(v: &mut [uint32x4_t; 16], m: &[uint32x4_t; 16], r: usize, rot8_ // Column step: G(0,4,8,12), G(1,5,9,13), G(2,6,10,14), G(3,7,11,15) // SAFETY: `s` contains fixed schedule indices in 0..16, and `m` is `[T; 16]`. - g4(v, 0, 4, 8, 12, *m.get_unchecked(s[0]), *m.get_unchecked(s[1]), rot8_tbl); - g4(v, 1, 5, 9, 13, *m.get_unchecked(s[2]), *m.get_unchecked(s[3]), rot8_tbl); g4( v, - 2, - 6, - 10, - 14, + [0, 4, 8, 12], + *m.get_unchecked(s[0]), + *m.get_unchecked(s[1]), + rot8_tbl, + ); + g4( + v, + [1, 5, 9, 13], + *m.get_unchecked(s[2]), + *m.get_unchecked(s[3]), + rot8_tbl, + ); + g4( + v, + [2, 6, 10, 14], *m.get_unchecked(s[4]), *m.get_unchecked(s[5]), rot8_tbl, ); g4( v, - 3, - 7, - 11, - 15, + [3, 7, 11, 15], *m.get_unchecked(s[6]), *m.get_unchecked(s[7]), rot8_tbl, @@ -661,40 +776,28 @@ unsafe fn round4(v: &mut [uint32x4_t; 16], m: &[uint32x4_t; 16], r: usize, rot8_ // Diagonal step: G(0,5,10,15), G(1,6,11,12), G(2,7,8,13), G(3,4,9,14) g4( v, - 0, - 5, - 10, - 15, + [0, 5, 10, 15], *m.get_unchecked(s[8]), *m.get_unchecked(s[9]), rot8_tbl, ); g4( v, - 1, - 6, - 11, - 12, + [1, 6, 11, 12], *m.get_unchecked(s[10]), *m.get_unchecked(s[11]), rot8_tbl, ); g4( v, - 2, - 7, - 8, - 13, + [2, 7, 8, 13], *m.get_unchecked(s[12]), *m.get_unchecked(s[13]), rot8_tbl, ); g4( v, - 3, - 4, - 9, - 14, + [3, 4, 9, 14], *m.get_unchecked(s[14]), *m.get_unchecked(s[15]), rot8_tbl, @@ -708,18 +811,17 @@ unsafe fn round4(v: &mut [uint32x4_t; 16], m: &[uint32x4_t; 16], r: usize, rot8_ /// /// # Safety /// -/// Caller must ensure NEON is available. +/// The current CPU must support NEON. Each input pointer must be readable for +/// `input_len` bytes, and `input_len` must be in `1..=CHUNK_LEN`. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] -pub(crate) unsafe fn hash4_neon( +unsafe fn hash4_neon( inputs: [*const u8; 4], input_len: usize, key: &[u32; 8], counter: u64, increment_counter: bool, - flags: u32, - flags_start: u32, - flags_end: u32, + flags: Hash4Flags, out: &mut [[u8; OUT_LEN]; 4], ) { debug_assert!(input_len > 0); @@ -730,10 +832,13 @@ pub(crate) unsafe fn hash4_neon( // can skip the generic load/pad path and use a tighter contiguous loader. let contiguous_full_chunks = input_len == CHUNK_LEN && (inputs[1] as usize).wrapping_sub(inputs[0] as usize) == CHUNK_LEN - && (inputs[2] as usize).wrapping_sub(inputs[0] as usize) == 2 * CHUNK_LEN - && (inputs[3] as usize).wrapping_sub(inputs[0] as usize) == 3 * CHUNK_LEN; + && (inputs[2] as usize).wrapping_sub(inputs[0] as usize) == CHUNK_LEN.strict_mul(2) + && (inputs[3] as usize).wrapping_sub(inputs[0] as usize) == CHUNK_LEN.strict_mul(3); #[inline(always)] + /// # Safety + /// + /// The current CPU must support NEON, and `dest` must be writable for 16 bytes. unsafe fn storeu_128(src: uint32x4_t, dest: *mut u8) { // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. @@ -763,41 +868,26 @@ pub(crate) unsafe fn hash4_neon( } else { (0u64, 0, 0, 0) }; - let counter_low_vec = vld1q_u32( - [ - (counter.wrapping_add(d0) as u32), - (counter.wrapping_add(d1) as u32), - (counter.wrapping_add(d2) as u32), - (counter.wrapping_add(d3) as u32), - ] - .as_ptr(), - ); - let counter_high_vec = vld1q_u32( - [ - ((counter.wrapping_add(d0) >> 32) as u32), - ((counter.wrapping_add(d1) >> 32) as u32), - ((counter.wrapping_add(d2) >> 32) as u32), - ((counter.wrapping_add(d3) >> 32) as u32), - ] - .as_ptr(), - ); + let (counter_low, counter_high) = counter_lanes(counter, [d0, d1, d2, d3]); + let counter_low_vec = vld1q_u32(counter_low.as_ptr()); + let counter_high_vec = vld1q_u32(counter_high.as_ptr()); // Process each block - let mut block_flags = flags | flags_start; + let mut block_flags = flags.base | flags.start; let rot8_tbl = vld1q_u8(ROT8_TABLE.as_ptr()); for block_idx in 0..num_blocks { - let block_offset = block_idx * BLOCK_LEN; - let is_last = block_idx == num_blocks - 1; + let block_offset = block_idx.strict_mul(BLOCK_LEN); + let is_last = block_idx == num_blocks.strict_sub(1); // Calculate block length for last block let block_len = if is_last && !input_len.is_multiple_of(BLOCK_LEN) { - (input_len % BLOCK_LEN) as u32 + u32::try_from(input_len % BLOCK_LEN).expect("a partial BLAKE3 block length fits in u32") } else { - BLOCK_LEN as u32 + BLOCK_LEN_U32 }; if is_last { - block_flags |= flags_end; + block_flags |= flags.end; } // Load and transpose message blocks (pads the last block if needed). @@ -846,7 +936,7 @@ pub(crate) unsafe fn hash4_neon( h_vecs[6] = veorq_u32(v[6], v[14]); h_vecs[7] = veorq_u32(v[7], v[15]); - block_flags = flags; + block_flags = flags.base; } // Transpose the CV vectors so we can store each output contiguously. @@ -855,7 +945,7 @@ pub(crate) unsafe fn hash4_neon( transpose_vecs(&mut lo); transpose_vecs(&mut hi); - for lane in 0..4 { + for lane in 0usize..4 { let dst = out[lane].as_mut_ptr(); storeu_128(lo[lane], dst.add(0)); storeu_128(hi[lane], dst.add(16)); @@ -867,6 +957,11 @@ pub(crate) unsafe fn hash4_neon( /// /// This is the hot-path worker used by `hash_many_contiguous_neon` to avoid /// generic variable-size handling and repeated contiguous-shape checks. +/// +/// # Safety +/// +/// The current CPU must support NEON. `input0` must be readable for four full +/// chunks, and `out` must be writable for four outputs. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] unsafe fn hash4_contiguous_full_chunks_neon_to_out( @@ -877,6 +972,9 @@ unsafe fn hash4_contiguous_full_chunks_neon_to_out( out: *mut u8, ) { #[inline(always)] + /// # Safety + /// + /// The current CPU must support NEON, and `dest` must be writable for 16 bytes. unsafe fn storeu_128(src: uint32x4_t, dest: *mut u8) { // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. @@ -899,34 +997,19 @@ unsafe fn hash4_contiguous_full_chunks_neon_to_out( vdupq_n_u32(key[7]), ]; - let counter_low_vec = vld1q_u32( - [ - counter as u32, - counter.wrapping_add(1) as u32, - counter.wrapping_add(2) as u32, - counter.wrapping_add(3) as u32, - ] - .as_ptr(), - ); - let counter_high_vec = vld1q_u32( - [ - (counter >> 32) as u32, - (counter.wrapping_add(1) >> 32) as u32, - (counter.wrapping_add(2) >> 32) as u32, - (counter.wrapping_add(3) >> 32) as u32, - ] - .as_ptr(), - ); + let (counter_low, counter_high) = counter_lanes(counter, [0, 1, 2, 3]); + let counter_low_vec = vld1q_u32(counter_low.as_ptr()); + let counter_high_vec = vld1q_u32(counter_high.as_ptr()); - let block_len_vec = vdupq_n_u32(BLOCK_LEN as u32); + let block_len_vec = vdupq_n_u32(BLOCK_LEN_U32); let mut block_flags = flags | CHUNK_START; let rot8_tbl = vld1q_u8(ROT8_TABLE.as_ptr()); for block_idx in 0..(CHUNK_LEN / BLOCK_LEN) { - if block_idx + 1 == (CHUNK_LEN / BLOCK_LEN) { + if block_idx.strict_add(1) == (CHUNK_LEN / BLOCK_LEN) { block_flags |= super::CHUNK_END; } - let msg = load_msg_vecs_transposed_contiguous(input0, block_idx * BLOCK_LEN); + let msg = load_msg_vecs_transposed_contiguous(input0, block_idx.strict_mul(BLOCK_LEN)); let block_flags_vec = vdupq_n_u32(block_flags); let mut v = [ @@ -973,8 +1056,8 @@ unsafe fn hash4_contiguous_full_chunks_neon_to_out( transpose_vecs(&mut lo); transpose_vecs(&mut hi); - for lane in 0..4 { - let dst = out.add(lane * OUT_LEN); + for lane in 0usize..4 { + let dst = out.add(lane.strict_mul(OUT_LEN)); storeu_128(lo[lane], dst.add(0)); storeu_128(hi[lane], dst.add(16)); } @@ -1009,7 +1092,9 @@ pub(crate) unsafe fn root_hash_one_chunk_root_aarch64(input: *const u8, key: &[u let mut out = [0u8; OUT_LEN]; for (j, word) in out_words.iter().copied().enumerate() { - out[j * 4..j * 4 + 4].copy_from_slice(&word.to_le_bytes()); + let start = j.strict_mul(4); + let end = start.strict_add(4); + out[start..end].copy_from_slice(&word.to_le_bytes()); } return out; } @@ -1056,29 +1141,20 @@ pub(crate) unsafe fn chunk_cv_one_chunk_aarch64_out( // Fallback: per-block NEON compressor. let mut cv = *key; for block_idx in 0..(CHUNK_LEN / BLOCK_LEN) { - let block_bytes: &[u8; BLOCK_LEN] = { - let src = input.add(block_idx * BLOCK_LEN); - &*(src as *const [u8; BLOCK_LEN]) - }; + let block = input.add(block_idx.strict_mul(BLOCK_LEN)); let start = if block_idx == 0 { CHUNK_START } else { 0 }; - let end = if block_idx + 1 == (CHUNK_LEN / BLOCK_LEN) { + let end = if block_idx.strict_add(1) == (CHUNK_LEN / BLOCK_LEN) { super::CHUNK_END } else { 0 }; - cv = compress_cv_neon_bytes( - &cv, - block_bytes.as_ptr(), - counter, - BLOCK_LEN as u32, - flags | start | end, - ); + cv = compress_cv_neon_bytes(&cv, block, counter, BLOCK_LEN_U32, flags | start | end); } for (j, &word) in cv.iter().enumerate() { let bytes = word.to_le_bytes(); - core::ptr::copy_nonoverlapping(bytes.as_ptr(), out.add(j * 4), 4); + core::ptr::copy_nonoverlapping(bytes.as_ptr(), out.add(j.strict_mul(4)), 4); } } } @@ -1169,21 +1245,18 @@ pub(crate) unsafe fn chunk_state_one_chunk_aarch64_out( // Fallback: per-block NEON compressor for blocks 0..14, then copy the final block bytes. let mut cv = *key; - for block_idx in 0..15 { - let block_bytes: &[u8; BLOCK_LEN] = { - let src = input.add(block_idx * BLOCK_LEN); - &*(src as *const [u8; BLOCK_LEN]) - }; + for block_idx in 0usize..15 { + let block = input.add(block_idx.strict_mul(BLOCK_LEN)); let start = if block_idx == 0 { CHUNK_START } else { 0 }; - cv = compress_cv_neon_bytes(&cv, block_bytes.as_ptr(), counter, BLOCK_LEN as u32, flags | start); + cv = compress_cv_neon_bytes(&cv, block, counter, BLOCK_LEN_U32, flags | start); } // Store cv. core::ptr::copy_nonoverlapping(cv.as_ptr(), out_cv, 8); // Copy final block bytes. - core::ptr::copy_nonoverlapping(input.add(15 * BLOCK_LEN), out_last_block, BLOCK_LEN); + core::ptr::copy_nonoverlapping(input.add(15usize.strict_mul(BLOCK_LEN)), out_last_block, BLOCK_LEN); } } @@ -1212,9 +1285,11 @@ pub(crate) unsafe fn root_hash_one_chunk_neon(input: *const u8, key: &[u32; 8], key, 0, false, - flags, - CHUNK_START, - super::CHUNK_END | super::ROOT, + Hash4Flags { + base: flags, + start: CHUNK_START, + end: super::CHUNK_END | super::ROOT, + }, &mut out, ); out[0] @@ -1247,16 +1322,17 @@ pub(crate) unsafe fn hash_many_contiguous_neon( // Process 8 chunks per loop iteration when possible: two specialized 4-chunk // kernels with fixed contiguous/full-chunk shape. - while idx + 8 <= num_chunks { + while num_chunks.strict_sub(idx) >= 8 { // SAFETY: caller guarantees `input` is valid for `num_chunks * CHUNK_LEN`. - let src0 = input.add(idx * CHUNK_LEN); + let src0 = input.add(idx.strict_mul(CHUNK_LEN)); // SAFETY: caller guarantees `out` is valid for `num_chunks * OUT_LEN`. - let dst0 = out.add(idx * OUT_LEN); + let dst0 = out.add(idx.strict_mul(OUT_LEN)); // Prefetch next batch while processing current. - if idx + 16 <= num_chunks { + if num_chunks.strict_sub(idx) >= 16 { // SAFETY: prefetch is a CPU hint — invalid addresses are silently ignored by hardware. - crate::hashes::common::prefetch::prefetch_read_l1(input.add((idx + 8) * CHUNK_LEN)); + let next = idx.strict_add(8).strict_mul(CHUNK_LEN); + crate::hashes::common::prefetch::prefetch_read_l1(input.add(next)); } hash4_contiguous_full_chunks_neon_to_out(src0, key, counter, flags, dst0); @@ -1267,24 +1343,25 @@ pub(crate) unsafe fn hash_many_contiguous_neon( flags, dst0.add(4 * OUT_LEN), ); - idx += 8; + idx = idx.strict_add(8); counter = counter.wrapping_add(8); } - while idx + 4 <= num_chunks { + while num_chunks.strict_sub(idx) >= 4 { // SAFETY: caller guarantees `input` is valid for `num_chunks * CHUNK_LEN`. - let src = input.add(idx * CHUNK_LEN); + let src = input.add(idx.strict_mul(CHUNK_LEN)); // SAFETY: caller guarantees `out` is valid for `num_chunks * OUT_LEN`. - let dst = out.add(idx * OUT_LEN); + let dst = out.add(idx.strict_mul(OUT_LEN)); // Prefetch next batch while processing current. - if idx + 8 <= num_chunks { + if num_chunks.strict_sub(idx) >= 8 { // SAFETY: prefetch is a CPU hint — invalid addresses are silently ignored by hardware. - crate::hashes::common::prefetch::prefetch_read_l1(input.add((idx + 4) * CHUNK_LEN)); + let next = idx.strict_add(4).strict_mul(CHUNK_LEN); + crate::hashes::common::prefetch::prefetch_read_l1(input.add(next)); } hash4_contiguous_full_chunks_neon_to_out(src, key, counter, flags, dst); - idx += 4; + idx = idx.strict_add(4); counter = counter.wrapping_add(4); } @@ -1300,9 +1377,10 @@ pub(crate) unsafe fn hash_many_contiguous_neon( for lane in 0..remaining { // SAFETY: `lane < remaining <= num_chunks - idx`, so per-lane source and // destination pointers are within caller-provided buffers. - let src = input.add((idx + lane) * CHUNK_LEN); - let dst = out.add((idx + lane) * OUT_LEN); - let chunk_counter = counter.wrapping_add(lane as u64); + let item = idx.strict_add(lane); + let src = input.add(item.strict_mul(CHUNK_LEN)); + let dst = out.add(item.strict_mul(OUT_LEN)); + let chunk_counter = counter.wrapping_add(u64::try_from(lane).expect("BLAKE3 tail lane fits in u64")); chunk_cv_one_chunk_aarch64_out(src, key, chunk_counter, flags, dst); } } @@ -1313,6 +1391,11 @@ pub(crate) unsafe fn hash_many_contiguous_neon( /// `children` is interpreted as `[left0, right0, left1, right1, ...]`. /// This path is alignment-agnostic and is the canonical aarch64 parent combine /// fast path (asm is only used for chunk-state loops). +/// +/// # Safety +/// +/// The current CPU must support NEON. `children.len()` must equal +/// `2 * out.len()`. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] pub(crate) unsafe fn parent_cvs_many_neon( @@ -1321,40 +1404,62 @@ pub(crate) unsafe fn parent_cvs_many_neon( flags: u32, out: &mut [[u8; OUT_LEN]], ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if out.is_empty() { return; } let parent_flags = PARENT | flags; - debug_assert!(parent_flags <= u8::MAX as u32); // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. unsafe { let mut idx = 0usize; while idx < out.len() { - let rem = core::cmp::min(4usize, out.len() - idx); - let last_ptr = children[2 * (idx + rem - 1)].as_ptr(); + let rem = core::cmp::min(4usize, out.len().strict_sub(idx)); + let last_child = idx.strict_add(rem).strict_sub(1).strict_mul(2); + let last_ptr = children[last_child].as_ptr(); let mut ptrs = [last_ptr; 4]; - for lane in 0..rem { - ptrs[lane] = children[2 * (idx + lane)].as_ptr(); + for (lane, ptr) in ptrs.iter_mut().enumerate().take(rem) { + let child = idx.strict_add(lane).strict_mul(2); + *ptr = children[child].as_ptr(); } if rem == 4 { - // SAFETY: - // - `idx + 4 <= out.len()` by `rem == 4`. - // - `out` is `[[u8; OUT_LEN]]`, so taking a 4-element window as `[[u8; OUT_LEN]; 4]` is - // layout-compatible. - // - We write exactly those 4 outputs. - let out4 = &mut *(out.as_mut_ptr().add(idx) as *mut [[u8; OUT_LEN]; 4]); - hash4_neon(ptrs, BLOCK_LEN, &key_words, 0, false, parent_flags, 0, 0, out4); + let out4 = out[idx..] + .first_chunk_mut::<4>() + .expect("four remaining BLAKE3 parents provide four output slots"); + hash4_neon( + ptrs, + BLOCK_LEN, + &key_words, + 0, + false, + Hash4Flags { + base: parent_flags, + start: 0, + end: 0, + }, + out4, + ); } else { let mut tmp = [[0u8; OUT_LEN]; 4]; - hash4_neon(ptrs, BLOCK_LEN, &key_words, 0, false, parent_flags, 0, 0, &mut tmp); - out[idx..idx + rem].copy_from_slice(&tmp[..rem]); + hash4_neon( + ptrs, + BLOCK_LEN, + &key_words, + 0, + false, + Hash4Flags { + base: parent_flags, + start: 0, + end: 0, + }, + &mut tmp, + ); + out[idx..idx.strict_add(rem)].copy_from_slice(&tmp[..rem]); } - idx += rem; + idx = idx.strict_add(rem); } } } @@ -1365,7 +1470,8 @@ pub(crate) unsafe fn parent_cvs_many_neon( /// /// # Safety /// -/// Caller must ensure NEON is available. +/// The current CPU must support NEON. `blocks` must contain only complete +/// BLAKE3 blocks, and `blocks_compressed` must remain within one chunk. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] pub(crate) unsafe fn chunk_compress_blocks_neon( @@ -1395,7 +1501,7 @@ pub(crate) unsafe fn chunk_compress_blocks_neon( chaining_value.as_mut_ptr(), chunk_counter, flags, - blocks_compressed as *mut u8, + core::ptr::from_mut(blocks_compressed), num_blocks, ); return; @@ -1410,10 +1516,10 @@ pub(crate) unsafe fn chunk_compress_blocks_neon( chaining_value, block_bytes.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } } @@ -1422,6 +1528,11 @@ pub(crate) unsafe fn chunk_compress_blocks_neon( /// /// Returns the 8-word chaining value result (row0^row2, row1^row3), avoiding /// materializing the full 16-word compression output. +/// +/// # Safety +/// +/// The current CPU must support NEON, and `block` must be readable for +/// `BLOCK_LEN` bytes. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] pub(crate) unsafe fn compress_cv_neon_bytes( @@ -1469,7 +1580,7 @@ pub(crate) unsafe fn parent_cv_neon( &key_words, block_words.as_ptr().cast(), 0, - BLOCK_LEN as u32, + BLOCK_LEN_U32, PARENT | flags, ) } @@ -1494,6 +1605,9 @@ pub(crate) unsafe fn root_output_blocks4_neon( out: *mut u8, ) { #[inline(always)] + /// # Safety + /// + /// The current CPU must support NEON, and `dest` must be writable for 16 bytes. unsafe fn storeu_128(src: uint32x4_t, dest: *mut u8) { // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. @@ -1533,24 +1647,9 @@ pub(crate) unsafe fn root_output_blocks4_neon( vdupq_n_u32(block_words[15]), ]; - let counter_low_vec = vld1q_u32( - [ - counter as u32, - counter.wrapping_add(1) as u32, - counter.wrapping_add(2) as u32, - counter.wrapping_add(3) as u32, - ] - .as_ptr(), - ); - let counter_high_vec = vld1q_u32( - [ - (counter >> 32) as u32, - (counter.wrapping_add(1) >> 32) as u32, - (counter.wrapping_add(2) >> 32) as u32, - (counter.wrapping_add(3) >> 32) as u32, - ] - .as_ptr(), - ); + let (counter_low, counter_high) = counter_lanes(counter, [0, 1, 2, 3]); + let counter_low_vec = vld1q_u32(counter_low.as_ptr()); + let counter_high_vec = vld1q_u32(counter_high.as_ptr()); let block_len_vec = vdupq_n_u32(block_len); let flags_vec = vdupq_n_u32(flags); @@ -1616,8 +1715,8 @@ pub(crate) unsafe fn root_output_blocks4_neon( transpose_vecs(&mut g2); transpose_vecs(&mut g3); - for lane in 0..4 { - let base = out.add(lane * 64); + for lane in 0usize..4 { + let base = out.add(lane.strict_mul(BLOCK_LEN)); storeu_128(g0[lane], base); storeu_128(g1[lane], base.add(16)); storeu_128(g2[lane], base.add(32)); diff --git a/src/hashes/crypto/blake3/aarch64/asm.rs b/src/hashes/crypto/blake3/aarch64/asm.rs index da10523e..432c235a 100644 --- a/src/hashes/crypto/blake3/aarch64/asm.rs +++ b/src/hashes/crypto/blake3/aarch64/asm.rs @@ -9,8 +9,6 @@ //! - last-block byte outputs require 8-byte alignment //! - unaligned callers must use the NEON paths in `aarch64.rs`. -#![allow(unsafe_code)] - use core::arch::global_asm; pub(crate) const ASM_ALIGN_INPUT: usize = 8; @@ -25,13 +23,13 @@ global_asm!(include_str!("asm/rscrypto_blake3_hash1_chunk_aarch64_apple_darwin.s #[cfg(target_os = "linux")] unsafe extern "C" { - pub fn rscrypto_blake3_hash1_chunk_root_aarch64_unix_linux( + pub(super) fn rscrypto_blake3_hash1_chunk_root_aarch64_unix_linux( input: *const u8, key: *const u32, flags: u32, out: *mut u8, ); - pub fn rscrypto_blake3_hash1_chunk_cv_aarch64_unix_linux( + pub(super) fn rscrypto_blake3_hash1_chunk_cv_aarch64_unix_linux( input: *const u8, key: *const u32, counter: u64, @@ -39,7 +37,7 @@ unsafe extern "C" { out: *mut u8, ); - pub fn rscrypto_blake3_hash1_chunk_state_aarch64_unix_linux( + pub(super) fn rscrypto_blake3_hash1_chunk_state_aarch64_unix_linux( input: *const u8, key: *const u32, counter: u64, @@ -48,7 +46,7 @@ unsafe extern "C" { out_last_block: *mut u8, ); - pub fn rscrypto_blake3_chunk_compress_blocks_aarch64_unix_linux( + pub(super) fn rscrypto_blake3_chunk_compress_blocks_aarch64_unix_linux( blocks: *const u8, chaining_value: *mut u32, chunk_counter: u64, @@ -60,13 +58,13 @@ unsafe extern "C" { #[cfg(target_os = "macos")] unsafe extern "C" { - pub fn rscrypto_blake3_hash1_chunk_root_aarch64_apple_darwin( + pub(super) fn rscrypto_blake3_hash1_chunk_root_aarch64_apple_darwin( input: *const u8, key: *const u32, flags: u32, out: *mut u8, ); - pub fn rscrypto_blake3_hash1_chunk_cv_aarch64_apple_darwin( + pub(super) fn rscrypto_blake3_hash1_chunk_cv_aarch64_apple_darwin( input: *const u8, key: *const u32, counter: u64, @@ -74,7 +72,7 @@ unsafe extern "C" { out: *mut u8, ); - pub fn rscrypto_blake3_hash1_chunk_state_aarch64_apple_darwin( + pub(super) fn rscrypto_blake3_hash1_chunk_state_aarch64_apple_darwin( input: *const u8, key: *const u32, counter: u64, @@ -83,7 +81,7 @@ unsafe extern "C" { out_last_block: *mut u8, ); - pub fn rscrypto_blake3_chunk_compress_blocks_aarch64_apple_darwin( + pub(super) fn rscrypto_blake3_chunk_compress_blocks_aarch64_apple_darwin( blocks: *const u8, chaining_value: *mut u32, chunk_counter: u64, diff --git a/src/hashes/crypto/blake3/control.rs b/src/hashes/crypto/blake3/control.rs index ea45b042..95401f12 100644 --- a/src/hashes/crypto/blake3/control.rs +++ b/src/hashes/crypto/blake3/control.rs @@ -25,7 +25,7 @@ fn compute_derive_context_key_words(context: &str) -> [u32; 8] { } #[inline] -#[cfg_attr(feature = "std", allow(dead_code))] +#[cfg(not(feature = "std"))] pub(super) fn derive_context_key_words(context: &str) -> [u32; 8] { let context_bytes = context.as_bytes(); let kernel_ctx = dispatch::hasher_dispatch().size_class_kernel(context_bytes.len()); @@ -171,23 +171,23 @@ pub(super) fn streaming_parallel_threads_for_flags( #[cfg(feature = "parallel")] #[inline] -#[allow(clippy::manual_saturating_arithmetic)] // Explicit clamp semantics are preferred here. fn clamp_add_usize(lhs: usize, rhs: usize) -> usize { - lhs.checked_add(rhs).unwrap_or(usize::MAX) + lhs.saturating_add(rhs) } #[cfg(feature = "parallel")] #[inline] -#[allow(clippy::manual_saturating_arithmetic)] // Explicit clamp semantics are preferred here. fn clamp_mul_usize(lhs: usize, rhs: usize) -> usize { - lhs.checked_mul(rhs).unwrap_or(usize::MAX) + lhs.saturating_mul(rhs) } #[cfg(feature = "parallel")] #[inline] fn ceil_div_usize(value: usize, divisor: usize) -> usize { let adjusted = clamp_add_usize(value, divisor.strict_sub(1)); - adjusted / divisor + adjusted + .checked_div(divisor) + .expect("parallel policy divisor is nonzero") } #[cfg(feature = "parallel")] @@ -252,7 +252,7 @@ pub(super) fn parallel_admission_decision( threads: candidate, }; } - candidate -= 1; + candidate = candidate.strict_sub(1); } ParallelAdmissionDecision { would_parallelize: false, @@ -270,7 +270,10 @@ fn parallel_merge_divisor(mode: ParallelPolicyKind, commit_full_chunks: usize, t ParallelPolicyKind::Update | ParallelPolicyKind::KeyedUpdate | ParallelPolicyKind::DeriveUpdate => 1, ParallelPolicyKind::Xof | ParallelPolicyKind::KeyedXof | ParallelPolicyKind::DeriveXof => 3, }; - 1 + chunk_depth + thread_depth + mode_bias + 1usize + .strict_add(chunk_depth) + .strict_add(thread_depth) + .strict_add(mode_bias) } #[cfg(feature = "parallel")] diff --git a/src/hashes/crypto/blake3/dispatch.rs b/src/hashes/crypto/blake3/dispatch.rs index d64cfd0e..6d6fe5b1 100644 --- a/src/hashes/crypto/blake3/dispatch.rs +++ b/src/hashes/crypto/blake3/dispatch.rs @@ -1,5 +1,3 @@ -#![cfg_attr(test, allow(dead_code))] - #[cfg(feature = "parallel")] use super::dispatch_tables::ParallelTable; #[cfg(any( @@ -64,7 +62,10 @@ struct ResolvedDispatch { avx2_hash_many_one_chunk_fast_path: bool, #[cfg(target_arch = "x86_64")] hash_many_wide_pipeline: bool, - #[cfg(target_arch = "x86_64")] + #[cfg(all( + target_arch = "x86_64", + any(target_os = "linux", target_os = "macos", target_os = "windows") + ))] avx2_available: bool, } @@ -269,7 +270,10 @@ fn resolved() -> ResolvedDispatch { avx2_hash_many_one_chunk_fast_path: allow_avx2_hash_many_one_chunk_fast_path(caps), #[cfg(target_arch = "x86_64")] hash_many_wide_pipeline: is_wide_pipeline_for_hash_many(caps), - #[cfg(target_arch = "x86_64")] + #[cfg(all( + target_arch = "x86_64", + any(target_os = "linux", target_os = "macos", target_os = "windows") + ))] avx2_available: caps.has(required_caps(Blake3KernelId::X86Avx2)), } }) @@ -303,17 +307,17 @@ fn select(d: &ActiveDispatch, len: usize) -> Entry { } } -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] #[inline] #[must_use] -pub fn kernel_name_for_len(len: usize) -> &'static str { +pub(crate) fn kernel_name_for_len(len: usize) -> &'static str { let d = active(); select(&d, len).kernel.name } #[inline] #[must_use] -pub fn xof(data: &[u8]) -> super::Blake3XofReader { +pub(crate) fn xof(data: &[u8]) -> super::Blake3XofReader { let d = active(); let kernel = select(&d, data.len()).kernel; @@ -360,7 +364,10 @@ pub(crate) fn hash_many_wide_pipeline() -> bool { resolved().hash_many_wide_pipeline } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(target_os = "linux", target_os = "macos", target_os = "windows") +))] #[inline] #[must_use] pub(crate) fn avx2_available() -> bool { diff --git a/src/hashes/crypto/blake3/dispatch_tables.rs b/src/hashes/crypto/blake3/dispatch_tables.rs index 75919a20..1fe518c6 100644 --- a/src/hashes/crypto/blake3/dispatch_tables.rs +++ b/src/hashes/crypto/blake3/dispatch_tables.rs @@ -2,7 +2,7 @@ //! //! This module stores compact per-family profiles used by runtime dispatch. -pub use super::kernels::Blake3KernelId as KernelId; +pub(crate) use super::kernels::Blake3KernelId as KernelId; use crate::platform::Caps; #[cfg(target_arch = "aarch64")] use crate::platform::caps::aarch64; @@ -15,7 +15,7 @@ use crate::platform::caps::s390x; #[cfg(target_arch = "x86_64")] use crate::platform::caps::x86; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; // Architecture-specific thresholds for when to switch from table bulk kernel to size-class // selection. These are tuned based on SIMD width and latency characteristics. @@ -28,12 +28,19 @@ const THRESHOLD_NEON: usize = 16 * 1024; // NEON - slightly higher due to differ #[cfg(not(target_arch = "x86_64"))] const THRESHOLD_PORTABLE: usize = 32 * 1024; // Conservative for scalar +#[cfg(feature = "parallel")] const DEFAULT_PAR_SPAWN_COST_BYTES: usize = 24 * 1024; +#[cfg(feature = "parallel")] const DEFAULT_PAR_MERGE_COST_BYTES: usize = 16 * 1024; +#[cfg(feature = "parallel")] const DEFAULT_PAR_BYTES_PER_CORE_SMALL: usize = 256 * 1024; +#[cfg(feature = "parallel")] const DEFAULT_PAR_BYTES_PER_CORE_MEDIUM: usize = 128 * 1024; +#[cfg(feature = "parallel")] const DEFAULT_PAR_BYTES_PER_CORE_LARGE: usize = 64 * 1024; +#[cfg(feature = "parallel")] const DEFAULT_PAR_SMALL_LIMIT_BYTES: usize = 256 * 1024; +#[cfg(feature = "parallel")] const DEFAULT_PAR_MEDIUM_LIMIT_BYTES: usize = 2 * 1024 * 1024; /// Parallel hashing policy for large inputs (std-only). @@ -41,9 +48,9 @@ const DEFAULT_PAR_MEDIUM_LIMIT_BYTES: usize = 2 * 1024 * 1024; /// The scheduler treats this as an explicit cost model: /// - fixed terms: `spawn_cost_bytes`, `merge_cost_bytes` /// - work terms: `bytes_per_core_*` for small/medium/large payload classes -#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +#[cfg(feature = "parallel")] #[derive(Clone, Copy, Debug)] -pub struct ParallelTable { +pub(crate) struct ParallelTable { /// Minimum total input bytes before parallel hashing is considered. pub min_bytes: usize, /// Minimum number of full chunks to commit in one batch. @@ -76,14 +83,14 @@ pub struct ParallelTable { /// `bulk_sizeclass_threshold` is the minimum input length to use size-class-based bulk kernel /// selection instead of the table's default bulk kernel. This is architecture-specific. #[derive(Clone, Copy, Debug)] -pub struct StreamingTable { +pub(crate) struct StreamingTable { pub stream: KernelId, pub bulk: KernelId, pub bulk_sizeclass_threshold: usize, } #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -92,12 +99,13 @@ pub struct DispatchTable { } /// Compact profile for one microarchitecture family. -#[cfg_attr(not(feature = "parallel"), allow(dead_code))] #[derive(Clone, Copy, Debug)] -pub struct FamilyProfile { +pub(crate) struct FamilyProfile { pub dispatch: DispatchTable, pub streaming: StreamingTable, + #[cfg(feature = "parallel")] pub parallel: ParallelTable, + #[cfg(feature = "parallel")] pub streaming_parallel: ParallelTable, } @@ -216,6 +224,7 @@ const DEFAULT_STREAM_KERNEL: KernelId = KernelId::Portable; #[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] const DEFAULT_BULK_KERNEL: KernelId = SIMD_KERNEL; +#[cfg(feature = "parallel")] #[derive(Clone, Copy, Debug)] struct ParallelCostModel { spawn_cost_bytes: usize, @@ -227,6 +236,7 @@ struct ParallelCostModel { medium_limit_bytes: usize, } +#[cfg(feature = "parallel")] #[inline] #[must_use] const fn parallel_cost_model( @@ -249,6 +259,7 @@ const fn parallel_cost_model( } } +#[cfg(feature = "parallel")] #[inline] #[must_use] const fn parallel_table( @@ -271,7 +282,7 @@ const fn parallel_table( } } -#[cfg(any(target_arch = "s390x", target_arch = "powerpc64"))] +#[cfg(all(feature = "parallel", any(target_arch = "s390x", target_arch = "powerpc64")))] macro_rules! parallel_costs { ( $min_bytes:expr, @@ -302,6 +313,7 @@ macro_rules! parallel_costs { }; } +#[cfg(feature = "parallel")] #[inline] #[must_use] const fn default_parallel_costs(min_bytes: usize, min_chunks: usize, max_threads: u8) -> ParallelTable { @@ -323,7 +335,7 @@ const fn default_parallel_costs(min_bytes: usize, min_chunks: usize, max_threads #[inline] #[must_use] -#[cfg(any(target_arch = "s390x", target_arch = "powerpc64"))] +#[cfg(all(feature = "parallel", any(target_arch = "s390x", target_arch = "powerpc64")))] const fn scalar_profile_parallel( min_bytes: usize, min_chunks: usize, @@ -431,14 +443,14 @@ const fn default_kind_streaming_table() -> StreamingTable { } } -#[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] +#[cfg(all(feature = "parallel", any(target_arch = "x86_64", target_arch = "riscv64")))] #[inline] #[must_use] const fn default_kind_parallel_table() -> ParallelTable { default_parallel_costs(128 * 1024, 64, 0) } -#[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] +#[cfg(all(feature = "parallel", any(target_arch = "x86_64", target_arch = "riscv64")))] #[inline] #[must_use] const fn default_kind_streaming_parallel_table() -> ParallelTable { @@ -452,7 +464,9 @@ const fn default_kind_profile() -> FamilyProfile { FamilyProfile { dispatch: default_kind_table(), streaming: default_kind_streaming_table(), + #[cfg(feature = "parallel")] parallel: default_kind_parallel_table(), + #[cfg(feature = "parallel")] streaming_parallel: default_kind_streaming_parallel_table(), } } @@ -474,21 +488,20 @@ const fn portable_profile() -> FamilyProfile { bulk: KernelId::Portable, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: default_parallel_costs(128 * 1024, 64, 0), + #[cfg(feature = "parallel")] streaming_parallel: default_parallel_costs(128 * 1024, 64, 0), } } #[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] -// Family Profile: DEFAULT_KIND -pub static PROFILE_DEFAULT_KIND: FamilyProfile = default_kind_profile(); -// Family Profile: PORTABLE +pub(crate) static PROFILE_DEFAULT_KIND: FamilyProfile = default_kind_profile(); #[cfg(not(target_arch = "x86_64"))] -pub static PROFILE_PORTABLE: FamilyProfile = portable_profile(); +pub(crate) static PROFILE_PORTABLE: FamilyProfile = portable_profile(); -// Family Profile: INTEL_SAPPHIRE_RAPIDS #[cfg(target_arch = "x86_64")] -pub static PROFILE_INTEL_SAPPHIRE_RAPIDS: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_INTEL_SAPPHIRE_RAPIDS: FamilyProfile = FamilyProfile { dispatch: DispatchTable { boundaries: [64, 1024, 4096], // This profile uses AVX-512 for every size class; its selector has no @@ -503,6 +516,7 @@ pub static PROFILE_INTEL_SAPPHIRE_RAPIDS: FamilyProfile = FamilyProfile { bulk: KernelId::X86Avx512, bulk_sizeclass_threshold: THRESHOLD_AVX512, }, + #[cfg(feature = "parallel")] parallel: ParallelTable { min_bytes: 65536, min_chunks: 64, @@ -515,6 +529,7 @@ pub static PROFILE_INTEL_SAPPHIRE_RAPIDS: FamilyProfile = FamilyProfile { small_limit_bytes: 1048576, medium_limit_bytes: 4194304, }, + #[cfg(feature = "parallel")] streaming_parallel: ParallelTable { min_bytes: 0, min_chunks: 0, @@ -528,9 +543,8 @@ pub static PROFILE_INTEL_SAPPHIRE_RAPIDS: FamilyProfile = FamilyProfile { medium_limit_bytes: 2097152, }, }; -// Family Profile: X86_AVX512 #[cfg(target_arch = "x86_64")] -pub static PROFILE_X86_AVX512: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_X86_AVX512: FamilyProfile = FamilyProfile { dispatch: DispatchTable { boundaries: [64, 1024, 4096], // This profile uses AVX-512 for every size class; its selector has no @@ -545,6 +559,7 @@ pub static PROFILE_X86_AVX512: FamilyProfile = FamilyProfile { bulk: KernelId::X86Avx512, bulk_sizeclass_threshold: THRESHOLD_AVX512, }, + #[cfg(feature = "parallel")] parallel: ParallelTable { min_bytes: 65536, min_chunks: 64, @@ -557,6 +572,7 @@ pub static PROFILE_X86_AVX512: FamilyProfile = FamilyProfile { small_limit_bytes: 262144, medium_limit_bytes: 2097152, }, + #[cfg(feature = "parallel")] streaming_parallel: ParallelTable { min_bytes: 0, min_chunks: 0, @@ -572,7 +588,7 @@ pub static PROFILE_X86_AVX512: FamilyProfile = FamilyProfile { }; // Family Profile: AARCH64_NEON #[cfg(target_arch = "aarch64")] -pub static PROFILE_AARCH64_NEON: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_AARCH64_NEON: FamilyProfile = FamilyProfile { dispatch: DispatchTable { boundaries: [64, 4095, 4096], // Graviton-class attribution shows that for short inputs (<4KiB), the @@ -589,6 +605,7 @@ pub static PROFILE_AARCH64_NEON: FamilyProfile = FamilyProfile { bulk: KernelId::Aarch64Neon, bulk_sizeclass_threshold: THRESHOLD_NEON, }, + #[cfg(feature = "parallel")] parallel: ParallelTable { min_bytes: 65536, min_chunks: 64, @@ -601,6 +618,7 @@ pub static PROFILE_AARCH64_NEON: FamilyProfile = FamilyProfile { small_limit_bytes: 262144, medium_limit_bytes: 2097152, }, + #[cfg(feature = "parallel")] streaming_parallel: ParallelTable { min_bytes: 0, min_chunks: 0, @@ -616,33 +634,37 @@ pub static PROFILE_AARCH64_NEON: FamilyProfile = FamilyProfile { }; // Family Profile: Z13 #[cfg(target_arch = "s390x")] -pub static PROFILE_Z13: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_Z13: FamilyProfile = FamilyProfile { dispatch: default_kind_table(), streaming: StreamingTable { stream: KernelId::Portable, bulk: S390X_VECTOR_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: scalar_profile_parallel(256 * 1024, 128, 8, 0), + #[cfg(feature = "parallel")] streaming_parallel: scalar_profile_parallel(256 * 1024, 128, 8, 0), }; // Family Profile: Z14 #[cfg(target_arch = "s390x")] -pub static PROFILE_Z14: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_Z14: FamilyProfile = FamilyProfile { dispatch: default_kind_table(), streaming: StreamingTable { stream: KernelId::Portable, bulk: S390X_VECTOR_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: scalar_profile_parallel(192 * 1024, 96, 8, 1), + #[cfg(feature = "parallel")] streaming_parallel: scalar_profile_parallel(192 * 1024, 96, 8, 1), }; // Family Profile: Z15 #[cfg(target_arch = "s390x")] -pub static PROFILE_Z15: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_Z15: FamilyProfile = FamilyProfile { dispatch: DispatchTable { boundaries: [64, 256, 4096], xs: KernelId::Portable, @@ -655,6 +677,7 @@ pub static PROFILE_Z15: FamilyProfile = FamilyProfile { bulk: S390X_VECTOR_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: ParallelTable { min_bytes: 65536, min_chunks: 64, @@ -667,6 +690,7 @@ pub static PROFILE_Z15: FamilyProfile = FamilyProfile { small_limit_bytes: 262144, medium_limit_bytes: 8388608, }, + #[cfg(feature = "parallel")] streaming_parallel: ParallelTable { min_bytes: 0, min_chunks: 0, @@ -682,46 +706,52 @@ pub static PROFILE_Z15: FamilyProfile = FamilyProfile { }; // Family Profile: POWER7 #[cfg(target_arch = "powerpc64")] -pub static PROFILE_POWER7: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_POWER7: FamilyProfile = FamilyProfile { dispatch: default_kind_table(), streaming: StreamingTable { stream: KernelId::Portable, bulk: POWER_VSX_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: scalar_profile_parallel(256 * 1024, 128, 8, 0), + #[cfg(feature = "parallel")] streaming_parallel: scalar_profile_parallel(256 * 1024, 128, 8, 0), }; // Family Profile: POWER8 #[cfg(target_arch = "powerpc64")] -pub static PROFILE_POWER8: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_POWER8: FamilyProfile = FamilyProfile { dispatch: default_kind_table(), streaming: StreamingTable { stream: KernelId::Portable, bulk: POWER_VSX_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: scalar_profile_parallel(192 * 1024, 96, 8, 1), + #[cfg(feature = "parallel")] streaming_parallel: scalar_profile_parallel(192 * 1024, 96, 8, 1), }; // Family Profile: POWER9 #[cfg(target_arch = "powerpc64")] -pub static PROFILE_POWER9: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_POWER9: FamilyProfile = FamilyProfile { dispatch: default_kind_table(), streaming: StreamingTable { stream: KernelId::Portable, bulk: POWER_VSX_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: scalar_profile_parallel(128 * 1024, 64, 16, 3), + #[cfg(feature = "parallel")] streaming_parallel: scalar_profile_parallel(128 * 1024, 64, 16, 3), }; // Family Profile: POWER10 #[cfg(target_arch = "powerpc64")] -pub static PROFILE_POWER10: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_POWER10: FamilyProfile = FamilyProfile { dispatch: DispatchTable { boundaries: [64, 256, 4096], xs: KernelId::Portable, @@ -734,6 +764,7 @@ pub static PROFILE_POWER10: FamilyProfile = FamilyProfile { bulk: POWER_VSX_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: ParallelTable { min_bytes: 65536, min_chunks: 64, @@ -746,6 +777,7 @@ pub static PROFILE_POWER10: FamilyProfile = FamilyProfile { small_limit_bytes: 262144, medium_limit_bytes: 8388608, }, + #[cfg(feature = "parallel")] streaming_parallel: ParallelTable { min_bytes: 0, min_chunks: 0, @@ -762,7 +794,7 @@ pub static PROFILE_POWER10: FamilyProfile = FamilyProfile { #[inline] #[must_use] -pub fn select_profile_for_caps(caps: Caps) -> &'static FamilyProfile { +pub(crate) fn select_profile_for_caps(caps: Caps) -> &'static FamilyProfile { #[cfg(target_arch = "x86_64")] { if caps.has(x86::AVX512_READY) { @@ -837,27 +869,27 @@ pub fn select_profile_for_caps(caps: Caps) -> &'static FamilyProfile { #[inline] #[must_use] -pub fn select_table_for_caps(caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_table_for_caps(caps: Caps) -> &'static DispatchTable { &select_profile_for_caps(caps).dispatch } #[inline] #[must_use] -pub fn select_streaming_table_for_caps(caps: Caps) -> &'static StreamingTable { +pub(crate) fn select_streaming_table_for_caps(caps: Caps) -> &'static StreamingTable { &select_profile_for_caps(caps).streaming } #[cfg(feature = "parallel")] #[inline] #[must_use] -pub fn select_parallel_table_for_caps(caps: Caps) -> &'static ParallelTable { +pub(crate) fn select_parallel_table_for_caps(caps: Caps) -> &'static ParallelTable { &select_profile_for_caps(caps).parallel } #[cfg(feature = "parallel")] #[inline] #[must_use] -pub fn select_streaming_parallel_table_for_caps(caps: Caps) -> &'static ParallelTable { +pub(crate) fn select_streaming_parallel_table_for_caps(caps: Caps) -> &'static ParallelTable { &select_profile_for_caps(caps).streaming_parallel } diff --git a/src/hashes/crypto/blake3/kernel_test.rs b/src/hashes/crypto/blake3/kernel_test.rs index 2dab9a5e..01c4cff6 100644 --- a/src/hashes/crypto/blake3/kernel_test.rs +++ b/src/hashes/crypto/blake3/kernel_test.rs @@ -25,7 +25,7 @@ const ALL: &[Blake3KernelId] = &[ ]; #[derive(Clone, Debug)] -pub struct KernelResult { +pub(super) struct KernelResult { pub digest: [u8; 32], } @@ -48,7 +48,7 @@ fn digest_with_kernel(id: Blake3KernelId, data: &[u8]) -> [u8; 32] { } #[must_use] -pub fn run_all_blake3_kernels(data: &[u8]) -> Vec { +pub(super) fn run_all_blake3_kernels(data: &[u8]) -> Vec { let caps = crate::platform::caps(); let mut out = Vec::with_capacity(ALL.len()); for &id in ALL { @@ -61,7 +61,7 @@ pub fn run_all_blake3_kernels(data: &[u8]) -> Vec { out } -pub fn verify_blake3_kernels(data: &[u8]) -> Result<(), &'static str> { +pub(super) fn verify_blake3_kernels(data: &[u8]) -> Result<(), &'static str> { let results = run_all_blake3_kernels(data); let Some(first) = results.first() else { return Ok(()); @@ -93,7 +93,9 @@ mod tests { } fn pattern(len: usize) -> Vec { - (0..len).map(|i| (i % 251) as u8).collect() + (0..len) + .map(|i| u8::try_from(i % 251).expect("test pattern byte fits in u8")) + .collect() } #[test] @@ -335,7 +337,9 @@ mod tests { for chunk_idx in 0..num_chunks { let base = chunk_idx * CHUNK_LEN; for i in 0..CHUNK_LEN { - input[base + i] = ((i % 251) as u8).wrapping_add(chunk_idx as u8); + let byte = u8::try_from(i % 251).expect("test pattern byte fits in u8"); + let chunk = u8::try_from(chunk_idx).expect("test chunk index fits in u8"); + input[base + i] = byte.wrapping_add(chunk); } } @@ -383,7 +387,9 @@ mod tests { for chunk_idx in 0..num_chunks { let base = chunk_idx * CHUNK_LEN; for i in 0..CHUNK_LEN { - input[base + i] = ((i % 251) as u8).wrapping_add(chunk_idx as u8); + let byte = u8::try_from(i % 251).expect("test pattern byte fits in u8"); + let chunk = u8::try_from(chunk_idx).expect("test chunk index fits in u8"); + input[base + i] = byte.wrapping_add(chunk); } } diff --git a/src/hashes/crypto/blake3/kernels.rs b/src/hashes/crypto/blake3/kernels.rs index eb68b377..70ad4f61 100644 --- a/src/hashes/crypto/blake3/kernels.rs +++ b/src/hashes/crypto/blake3/kernels.rs @@ -1,6 +1,15 @@ use super::{ BLOCK_LEN, CHUNK_LEN, CHUNK_START, OUT_LEN, PARENT, first_8_words, words8_from_le_bytes_32, words16_from_le_bytes_64, }; +#[cfg(any( + test, + feature = "diag", + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "s390x", + target_arch = "powerpc64", + target_arch = "riscv64" +))] use crate::platform::Caps; #[cfg(target_arch = "aarch64")] use crate::platform::caps::aarch64; @@ -13,6 +22,33 @@ use crate::platform::caps::s390x; #[cfg(target_arch = "x86_64")] use crate::platform::caps::x86; +const BLOCK_LEN_U32: u32 = 64; + +#[cfg(all( + target_arch = "x86_64", + any(target_os = "linux", target_os = "macos", target_os = "windows") +))] +#[inline(always)] +fn assembly_flags(flags: u32) -> u8 { + u8::try_from(flags).expect("BLAKE3 flags must fit the assembly ABI") +} + +#[cfg(any(target_arch = "s390x", target_arch = "powerpc64", target_arch = "riscv64"))] +#[inline(always)] +fn counter_words(counter: u64) -> (u32, u32) { + let [c0, c1, c2, c3, c4, c5, c6, c7] = counter.to_le_bytes(); + ( + u32::from_le_bytes([c0, c1, c2, c3]), + u32::from_le_bytes([c4, c5, c6, c7]), + ) +} + +#[cfg(any(target_arch = "s390x", target_arch = "powerpc64", target_arch = "riscv64"))] +#[inline(always)] +fn wrapping_add4(lhs: core::simd::u32x4, rhs: core::simd::u32x4) -> core::simd::u32x4 { + core::ops::Add::add(lhs, rhs) +} + // Kernel function types /// Core compression function (single block). @@ -61,8 +97,7 @@ pub(crate) struct Kernel { #[cfg(all(feature = "diag", target_arch = "x86_64"))] pub(crate) force_x86_avx512_exact_block_asm: bool, /// Kernel name for debugging/tuning. - #[cfg(any(test, feature = "diag"))] - #[cfg_attr(test, allow(dead_code))] + #[cfg(feature = "diag")] pub(crate) name: &'static str, } @@ -71,7 +106,7 @@ pub(crate) struct Kernel { #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Blake3KernelId { +pub(crate) enum Blake3KernelId { Portable = 0, #[cfg(target_arch = "x86_64")] X86Sse41 = 2, @@ -93,7 +128,7 @@ impl Blake3KernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "x86_64")] @@ -116,7 +151,7 @@ impl Blake3KernelId { /// Returns the SIMD degree for this kernel. #[inline] #[must_use] - pub const fn simd_degree(self) -> usize { + pub(crate) const fn simd_degree(self) -> usize { match self { Self::Portable => 1, #[cfg(target_arch = "x86_64")] @@ -153,7 +188,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { owned_x86_hash_many: false, #[cfg(all(feature = "diag", target_arch = "x86_64"))] force_x86_avx512_exact_block_asm: false, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "x86_64")] @@ -172,7 +207,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { owned_x86_hash_many: false, #[cfg(all(feature = "diag", target_arch = "x86_64"))] force_x86_avx512_exact_block_asm: false, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "x86_64")] @@ -191,7 +226,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { owned_x86_hash_many: false, #[cfg(all(feature = "diag", target_arch = "x86_64"))] force_x86_avx512_exact_block_asm: false, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "x86_64")] @@ -210,7 +245,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { owned_x86_hash_many: false, #[cfg(all(feature = "diag", target_arch = "x86_64"))] force_x86_avx512_exact_block_asm: false, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "aarch64")] @@ -223,7 +258,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { compress: compress_neon_wrapper, chunk_compress_blocks: chunk_compress_blocks_neon_wrapper, hash_many_contiguous: hash_many_contiguous_neon_wrapper, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "s390x")] @@ -232,7 +267,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { compress: compress_s390x_vector_wrapper, chunk_compress_blocks: chunk_compress_blocks_s390x_vector_wrapper, hash_many_contiguous: hash_many_contiguous_s390x_vector_wrapper, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "powerpc64")] @@ -241,7 +276,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { compress: compress_power_vsx_wrapper, chunk_compress_blocks: chunk_compress_blocks_power_vsx_wrapper, hash_many_contiguous: hash_many_contiguous_power_vsx_wrapper, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "riscv64")] @@ -250,7 +285,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { compress: compress_riscv_v_wrapper, chunk_compress_blocks: chunk_compress_blocks_riscv_v_wrapper, hash_many_contiguous: hash_many_contiguous_riscv_v_wrapper, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, } @@ -366,7 +401,7 @@ pub(crate) unsafe fn compress_block_asm_inline( chaining_value, block.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags, ); } @@ -379,7 +414,7 @@ pub(crate) unsafe fn compress_block_asm_inline( chaining_value, block.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags, ); } @@ -392,7 +427,7 @@ pub(crate) unsafe fn compress_block_asm_inline( chaining_value, block.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags, ); } @@ -408,12 +443,19 @@ pub(crate) unsafe fn compress_block_asm_inline( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags, )); } #[inline(always)] +/// Dispatches contiguous full chunks through a selected kernel. +/// +/// # Safety +/// +/// `input` must reference `num_chunks * CHUNK_LEN` readable bytes, `out` must +/// reference `num_chunks * OUT_LEN` writable bytes, and the CPU must support +/// every target feature required by `id`. pub(crate) unsafe fn hash_many_contiguous_inline( id: Blake3KernelId, input: *const u8, @@ -545,8 +587,8 @@ fn write_root_output_words(out: &mut [u8; 2 * OUT_LEN], words: &[u32; 16]) { } for (idx, word) in words.iter().copied().enumerate() { - let offset = idx * 4; - out[offset..offset + 4].copy_from_slice(&word.to_le_bytes()); + let offset = idx.strict_mul(4); + out[offset..offset.strict_add(4)].copy_from_slice(&word.to_le_bytes()); } } @@ -1039,12 +1081,12 @@ where { debug_assert!(rem != 0); debug_assert!(rem <= DEGREE); - let last_ptr = ptr_at(total - 1); + let last_ptr = ptr_at(total.strict_sub(1)); let mut ptrs = [last_ptr; DEGREE]; let mut lane = 0usize; while lane < rem { - ptrs[lane] = ptr_at(start + lane); - lane += 1; + ptrs[lane] = ptr_at(start.strict_add(lane)); + lane = lane.strict_add(1); } ptrs } @@ -1064,15 +1106,15 @@ fn reduce_parent_blocks_lanes( let mut tmp = [[0u8; OUT_LEN]; DEGREE]; let mut i = 0usize; while i < count { - let rem = core::cmp::min(DEGREE, count - i); + let rem = core::cmp::min(DEGREE, count.strict_sub(i)); let ptrs = parent_block_ptrs::(i, rem, count, &mut ptr_at); hash_many(&ptrs, rem, &mut tmp); let mut lane = 0usize; while lane < rem { - sink(i + lane, &tmp[lane]); - lane += 1; + sink(i.strict_add(lane), &tmp[lane]); + lane = lane.strict_add(1); } - i += rem; + i = i.strict_add(rem); } } @@ -1087,24 +1129,24 @@ pub(crate) fn diag_chunk_cvs_many_avx2_pair_from_bytes( debug_assert!(!input.is_empty()); debug_assert_eq!(input.len() % CHUNK_LEN, 0); let chunks = input.len() / CHUNK_LEN; - debug_assert_eq!(out.len(), chunks * OUT_LEN); + debug_assert_eq!(out.len(), chunks.strict_mul(OUT_LEN)); let mut idx = 0usize; - while idx + 1 < chunks { + while idx.strict_add(1) < chunks { // SAFETY: Diagnostic owned AVX2 two-chunk batch because: // 1. Diagnostic availability checks AVX2 support before this helper is called. // 2. `idx + 1 < chunks`, so `input + idx * CHUNK_LEN` is readable for two full chunks. // 3. `out + idx * OUT_LEN` is writable for two OUT_LEN-byte CV outputs. unsafe { super::x86_64::avx2::hash2_chunks_owned( - input.as_ptr().add(idx * CHUNK_LEN), + input.as_ptr().add(idx.strict_mul(CHUNK_LEN)), &key_words, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("chunk index must fit u64")), flags, - out.as_mut_ptr().add(idx * OUT_LEN), + out.as_mut_ptr().add(idx.strict_mul(OUT_LEN)), ); } - idx += 2; + idx = idx.strict_add(2); } if idx < chunks { @@ -1114,11 +1156,11 @@ pub(crate) fn diag_chunk_cvs_many_avx2_pair_from_bytes( // 3. Diagnostic availability checks AVX2 support before this helper is called. unsafe { hash_one_chunk_avx2_owned_serial( - input.as_ptr().add(idx * CHUNK_LEN), + input.as_ptr().add(idx.strict_mul(CHUNK_LEN)), &key_words, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("chunk index must fit u64")), flags, - out.as_mut_ptr().add(idx * OUT_LEN), + out.as_mut_ptr().add(idx.strict_mul(OUT_LEN)), ); } } @@ -1131,7 +1173,7 @@ pub(crate) fn diag_parent_cvs_many_avx2_owned_from_bytes( flags: u32, out: &mut [[u8; OUT_LEN]], ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if out.is_empty() { return; } @@ -1139,24 +1181,24 @@ pub(crate) fn diag_parent_cvs_many_avx2_owned_from_bytes( let parent_flags = PARENT | flags; reduce_parent_blocks_lanes::<{ super::x86_64::avx2::DEGREE }, _, _, _>( out.len(), - |idx| children[2 * idx].as_ptr(), + |idx| children[2usize.strict_mul(idx)].as_ptr(), |ptrs, _rem, tmp| { // SAFETY: Diagnostic owned AVX2 parent batch because: // 1. Diagnostic availability checks AVX2 support before this helper is called. // 2. `parent_block_ptrs` fills every lane with a valid 64-byte parent block pointer. // 3. `tmp` has space for all 8 output CV lanes; callers copy only real lanes. unsafe { - super::x86_64::avx2::hash8_owned( - ptrs, - 1, - &key_words, - 0, - false, - parent_flags, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::avx2::hash8_owned(super::x86_64::HashManyRequest { + inputs: ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } }, |idx, bytes| out[idx] = *bytes, @@ -1170,10 +1212,13 @@ pub(crate) fn diag_parent_cvs_many_avx2_pair_from_bytes( flags: u32, out: &mut [[u8; OUT_LEN]], ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); let mut idx = 0usize; - while idx + 1 < out.len() { - let parents = [children[2 * idx].as_ptr(), children[2 * (idx + 1)].as_ptr()]; + while idx.strict_add(1) < out.len() { + let parents = [ + children[2usize.strict_mul(idx)].as_ptr(), + children[2usize.strict_mul(idx.strict_add(1))].as_ptr(), + ]; // SAFETY: Diagnostic owned AVX2 two-parent batch because: // 1. Diagnostic availability checks AVX2 support before this helper is called. // 2. `parents` points to two adjacent 64-byte parent blocks. @@ -1181,7 +1226,7 @@ pub(crate) fn diag_parent_cvs_many_avx2_pair_from_bytes( unsafe { super::x86_64::avx2::parent_cv2_owned(&parents, &key_words, flags, out[idx].as_mut_ptr()); } - idx += 2; + idx = idx.strict_add(2); } if idx < out.len() { @@ -1190,12 +1235,26 @@ pub(crate) fn diag_parent_cvs_many_avx2_pair_from_bytes( // 2. `idx < out.len()`, so `out[idx]` is writable for one OUT_LEN-byte CV. // 3. Diagnostic availability checks AVX2 support before this helper is called. unsafe { - parent_one_avx2_owned_serial_from_block(children[2 * idx].as_ptr(), key_words, flags, &mut out[idx]); + parent_one_avx2_owned_serial_from_block( + children[2usize.strict_mul(idx)].as_ptr(), + key_words, + flags, + &mut out[idx], + ); } } } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows") +))] +/// Hash one packed parent block through the serial AVX2 path. +/// +/// # Safety +/// +/// AVX2 must be available, `parent_block` must be readable for 64 bytes, and +/// `out` must be writable for one chaining value. unsafe fn parent_one_avx2_owned_serial_from_block( parent_block: *const u8, key_words: [u32; 8], @@ -1240,7 +1299,7 @@ pub(crate) fn diag_parent_cvs_many_avx512_owned_from_bytes( flags: u32, out: &mut [[u8; OUT_LEN]], ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if out.is_empty() { return; } @@ -1248,24 +1307,24 @@ pub(crate) fn diag_parent_cvs_many_avx512_owned_from_bytes( let parent_flags = PARENT | flags; reduce_parent_blocks_lanes::<{ super::x86_64::avx512::DEGREE }, _, _, _>( out.len(), - |idx| children[2 * idx].as_ptr(), + |idx| children[2usize.strict_mul(idx)].as_ptr(), |ptrs, _rem, tmp| { // SAFETY: Diagnostic owned AVX-512 parent batch because: // 1. Diagnostic availability checks AVX-512F/VL/DQ plus AVX2 support before this helper is called. // 2. `parent_block_ptrs` fills every lane with a valid 64-byte parent block pointer. // 3. `tmp` has space for all 16 output CV lanes; callers copy only real lanes. unsafe { - super::x86_64::avx512::hash16_owned( - ptrs, - 1, - &key_words, - 0, - false, - parent_flags, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::avx512::hash16_owned(super::x86_64::HashManyRequest { + inputs: ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } }, |idx, bytes| out[idx] = *bytes, @@ -1284,13 +1343,13 @@ pub(crate) fn parent_cvs_many_from_cvs_inline( flags: u32, out: &mut [[u32; 8]], ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if out.is_empty() { return; } if id == Blake3KernelId::Portable { - for (pair, out_cv) in children.chunks_exact(2).zip(out.iter_mut()) { + for (pair, out_cv) in children.as_chunks::<2>().0.iter().zip(out.iter_mut()) { *out_cv = parent_cv_inline(id, pair[0], pair[1], key_words, flags); } return; @@ -1313,7 +1372,7 @@ pub(crate) fn parent_cvs_many_from_cvs_inline( #[cfg(not(target_endian = "little"))] { // Big-endian fallback keeps explicit LE conversion. - for (pair, out_cv) in children.chunks_exact(2).zip(out.iter_mut()) { + for (pair, out_cv) in children.as_chunks::<2>().0.iter().zip(out.iter_mut()) { *out_cv = parent_cv_inline(id, pair[0], pair[1], key_words, flags); } } @@ -1334,7 +1393,7 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( flags: u32, out: &mut [[u8; OUT_LEN]], ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if out.is_empty() { return; } @@ -1368,21 +1427,21 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( let parent_flags = PARENT | flags; reduce_parent_blocks_lanes::<{ super::x86_64::sse41::DEGREE }, _, _, _>( out.len(), - |idx| children[2 * idx].as_ptr(), + |idx| children[2usize.strict_mul(idx)].as_ptr(), |ptrs, _rem, tmp| { // SAFETY: SSE4.1 is available per dispatch; pointers and outputs are valid. unsafe { - super::x86_64::sse41::hash4( - ptrs, - 1, - &key_words, - 0, - false, - parent_flags, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::sse41::hash4(super::x86_64::HashManyRequest { + inputs: ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } }, |idx, bytes| out[idx] = *bytes, @@ -1395,10 +1454,9 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( { let parent_flags = PARENT | flags; const DEGREE: usize = 16; - debug_assert!(parent_flags <= u8::MAX as u32); reduce_parent_blocks_lanes::( out.len(), - |idx| children[2 * idx].as_ptr(), + |idx| children[2usize.strict_mul(idx)].as_ptr(), |ptrs, rem, tmp| { if rem == 15 && avx512_owned_hash_many_available() { // SAFETY: Use the owned AVX-512 parent tail because: @@ -1408,35 +1466,35 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // real parent block. // 4. `tmp` has space for all 16 OUT_LEN-byte outputs. unsafe { - super::x86_64::avx512::hash16_owned( - ptrs, - 1, - &key_words, - 0, - false, - parent_flags, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::avx512::hash16_owned(super::x86_64::HashManyRequest { + inputs: ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } return; } // SAFETY: AVX-512 is available per dispatch; pointers are valid for // one parent block each, `rem <= DEGREE`, and output is large enough. unsafe { - super::x86_64::asm::hash_many_avx512( - ptrs.as_ptr(), - rem, - 1, - key_words.as_ptr(), - 0, - false, - parent_flags as u8, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::asm::hash_many_avx512(super::x86_64::asm::HashManyRequest { + inputs: ptrs.as_ptr(), + num_inputs: rem, + blocks: 1, + key: key_words.as_ptr(), + counter: 0, + increment_counter: false, + flags: assembly_flags(parent_flags), + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } }, |idx, bytes| out[idx] = *bytes, @@ -1449,7 +1507,7 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // Keep AVX-512 semantics on non-asm targets as well: run the parent // fold through the AVX-512 per-parent entrypoint instead of delegating // into AVX2. - for (pair, out_cv) in children.chunks_exact(2).zip(out.iter_mut()) { + for (pair, out_cv) in children.as_chunks::<2>().0.iter().zip(out.iter_mut()) { let left = words8_from_le_bytes_32(&pair[0]); let right = words8_from_le_bytes_32(&pair[1]); *out_cv = super::words8_to_le_bytes(&parent_cv_inline( @@ -1516,25 +1574,24 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( parent_block_ptr_from_children(children, 2), parent_block_ptr_from_children(children, 3), ]; - debug_assert!(parent_flags <= u8::MAX as u32); // SAFETY: Keep the four-parent AVX2 assembly tail but write it directly because: // 1. Dispatch selected the AVX2 kernel, so AVX2 is available. // 2. `ptrs` points to four readable packed 64-byte parent blocks. // 3. `out` is writable for four contiguous OUT_LEN-byte parent CV outputs. // 4. Direct output avoids the generic reducer's temporary copy. unsafe { - super::x86_64::asm::hash_many_avx2( - ptrs.as_ptr(), - 4, - 1, - key_words.as_ptr(), - 0, - false, - parent_flags as u8, - 0, - 0, - out[0].as_mut_ptr(), - ); + super::x86_64::asm::hash_many_avx2(super::x86_64::asm::HashManyRequest { + inputs: ptrs.as_ptr(), + num_inputs: 4, + blocks: 1, + key: key_words.as_ptr(), + counter: 0, + increment_counter: false, + flags: assembly_flags(parent_flags), + flags_start: 0, + flags_end: 0, + out: out[0].as_mut_ptr(), + }); } return; } @@ -1551,7 +1608,17 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // 3. `out` is writable for 8 contiguous OUT_LEN-byte parent CV outputs. // 4. Direct output avoids the generic reducer's temporary copy. unsafe { - super::x86_64::avx2::hash8_owned(&ptrs, 1, &key_words, 0, false, parent_flags, 0, 0, out[0].as_mut_ptr()); + super::x86_64::avx2::hash8_owned(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: out[0].as_mut_ptr(), + }); } return; } @@ -1559,12 +1626,11 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( reduce_parent_blocks_lanes::<{ super::x86_64::avx2::DEGREE }, _, _, _>( out.len(), - |idx| children[2 * idx].as_ptr(), + |idx| children[2usize.strict_mul(idx)].as_ptr(), |ptrs, _rem, tmp| { #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] { let rem = _rem; - debug_assert!(parent_flags <= u8::MAX as u32); if rem == 1 { // SAFETY: Use the owned AVX2 one-parent tail because: // 1. Dispatch selected the AVX2 kernel, so AVX2 is available. @@ -1585,17 +1651,17 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // 4. The `rem` guard keeps 2/3/4 generic-reducer tails on assembly and promotes only 5/6/7, which // won on Sapphire Rapids. Exact 2/3/8 parent reductions use direct fast paths above. unsafe { - super::x86_64::avx2::hash8_owned( - ptrs, - 1, - &key_words, - 0, - false, - parent_flags, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::avx2::hash8_owned(super::x86_64::HashManyRequest { + inputs: ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } return; } @@ -1603,18 +1669,18 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // dispatch. Each pointer is valid for one 64-byte parent block. // `rem <= DEGREE`, and `tmp` is large enough. unsafe { - super::x86_64::asm::hash_many_avx2( - ptrs.as_ptr(), - rem, - 1, - key_words.as_ptr(), - 0, - false, - parent_flags as u8, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::asm::hash_many_avx2(super::x86_64::asm::HashManyRequest { + inputs: ptrs.as_ptr(), + num_inputs: rem, + blocks: 1, + key: key_words.as_ptr(), + counter: 0, + increment_counter: false, + flags: assembly_flags(parent_flags), + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } } @@ -1622,17 +1688,17 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( { // SAFETY: AVX2 is available for this wrapper; pointers and outputs are valid. unsafe { - super::x86_64::avx2::hash8( - ptrs, - 1, - &key_words, - 0, - false, - parent_flags, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ) + super::x86_64::avx2::hash8(super::x86_64::HashManyRequest { + inputs: ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }) }; } }, @@ -1643,7 +1709,7 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( } // Scalar fallback. - for (pair, out_cv) in children.chunks_exact(2).zip(out.iter_mut()) { + for (pair, out_cv) in children.as_chunks::<2>().0.iter().zip(out.iter_mut()) { let left = words8_from_le_bytes_32(&pair[0]); let right = words8_from_le_bytes_32(&pair[1]); *out_cv = super::words8_to_le_bytes(&parent_cv_inline(id, left, right, key_words, flags)); @@ -1656,7 +1722,16 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( /// `[left_cv (8 words), right_cv (8 words)]`. #[inline] #[must_use] -pub const fn required_caps(id: Blake3KernelId) -> Caps { +#[cfg(any( + test, + feature = "diag", + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "s390x", + target_arch = "powerpc64", + target_arch = "riscv64" +))] +pub(crate) const fn required_caps(id: Blake3KernelId) -> Caps { match id { Blake3KernelId::Portable => Caps::NONE, #[cfg(target_arch = "x86_64")] @@ -1702,7 +1777,7 @@ pub const fn required_caps(id: Blake3KernelId) -> Caps { #[cfg(all(feature = "diag", target_arch = "x86_64"))] #[inline] #[must_use] -pub const fn required_caps_owned_hash_many(id: Blake3KernelId) -> Caps { +pub(crate) const fn required_caps_owned_hash_many(id: Blake3KernelId) -> Caps { match id { Blake3KernelId::X86Avx2 => required_caps(Blake3KernelId::X86Avx2), Blake3KernelId::X86Avx512 => x86::AVX512F @@ -1718,7 +1793,7 @@ pub const fn required_caps_owned_hash_many(id: Blake3KernelId) -> Caps { #[cfg(all(feature = "diag", target_arch = "x86_64"))] #[inline] #[must_use] -pub const fn required_caps_owned_compress(id: Blake3KernelId) -> Caps { +pub(crate) const fn required_caps_owned_compress(id: Blake3KernelId) -> Caps { match id { Blake3KernelId::X86Avx512 => required_caps(Blake3KernelId::X86Avx512), _ => required_caps(id), @@ -1735,27 +1810,24 @@ fn chunk_compress_blocks_portable( blocks_compressed: &mut u8, blocks: &[u8], ) { - debug_assert_eq!(blocks.len() % BLOCK_LEN, 0); + let (block_slices, remainder) = blocks.as_chunks::(); + debug_assert!(remainder.is_empty()); // Hot path for streaming callers that feed one full block at a time. - if blocks.len() == BLOCK_LEN { - // SAFETY: `blocks` is exactly one block, and `[u8; BLOCK_LEN]` has 1-byte alignment. - let block_bytes: &[u8; BLOCK_LEN] = unsafe { &*(blocks.as_ptr().cast()) }; + if let [block_bytes] = block_slices { let start = if *blocks_compressed == 0 { CHUNK_START } else { 0 }; let block_words = words16_from_le_bytes_64(block_bytes); *chaining_value = first_8_words((super::compress)( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, )); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = (*blocks_compressed).strict_add(1); return; } - let (block_slices, remainder) = blocks.as_chunks::(); - debug_assert!(remainder.is_empty()); for block_bytes in block_slices { let start = if *blocks_compressed == 0 { CHUNK_START } else { 0 }; let block_words = words16_from_le_bytes_64(block_bytes); @@ -1763,10 +1835,10 @@ fn chunk_compress_blocks_portable( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, )); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = (*blocks_compressed).strict_add(1); } } @@ -1779,11 +1851,17 @@ fn parent_cv_portable(left_child_cv: [u32; 8], right_child_cv: [u32; 8], key_wor &key_words, &block_words, 0, - BLOCK_LEN as u32, + BLOCK_LEN_U32, PARENT | flags, )) } +/// Hashes contiguous chunks with the portable compression function. +/// +/// # Safety +/// +/// `input` must reference `num_chunks * CHUNK_LEN` readable bytes and `out` +/// must reference `num_chunks * OUT_LEN` writable bytes. unsafe fn hash_many_contiguous_portable( input: *const u8, num_chunks: usize, @@ -1795,13 +1873,17 @@ unsafe fn hash_many_contiguous_portable( debug_assert!(num_chunks != 0); for chunk_idx in 0..num_chunks { - let chunk_counter = counter.wrapping_add(chunk_idx as u64); + let chunk_counter = counter.wrapping_add(u64::try_from(chunk_idx).expect("BLAKE3 chunk index fits in u64")); let mut cv = *key; - for block_idx in 0..(CHUNK_LEN / BLOCK_LEN) { + for block_idx in 0..CHUNK_LEN.strict_div(BLOCK_LEN) { // SAFETY: caller guarantees `input` is valid for `num_chunks * CHUNK_LEN`. let block_words = unsafe { - let src = input.add(chunk_idx * CHUNK_LEN + block_idx * BLOCK_LEN); + let src = input.add( + chunk_idx + .strict_mul(CHUNK_LEN) + .strict_add(block_idx.strict_mul(BLOCK_LEN)), + ); // SAFETY: // - Caller guarantees `src` is valid for `BLOCK_LEN` bytes. // - `[u8; BLOCK_LEN]` has alignment 1, so this reference doesn't assume any alignment of the input @@ -1810,7 +1892,7 @@ unsafe fn hash_many_contiguous_portable( }; let start = if block_idx == 0 { CHUNK_START } else { 0 }; - let end = if block_idx + 1 == (CHUNK_LEN / BLOCK_LEN) { + let end = if block_idx.strict_add(1) == CHUNK_LEN.strict_div(BLOCK_LEN) { super::CHUNK_END } else { 0 @@ -1820,26 +1902,32 @@ unsafe fn hash_many_contiguous_portable( &cv, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, block_flags, )); } for (j, &word) in cv.iter().enumerate() { let bytes = word.to_le_bytes(); + let offset = chunk_idx.strict_mul(OUT_LEN).strict_add(j.strict_mul(4)); // SAFETY: caller guarantees out is valid for `num_chunks * OUT_LEN`. - unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), out.add(chunk_idx * OUT_LEN + j * 4), 4) }; + unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), out.add(offset), 4) }; } } } #[cfg(any(target_arch = "s390x", target_arch = "powerpc64", target_arch = "riscv64"))] #[inline(always)] +/// Writes one chaining value in little-endian byte order. +/// +/// # Safety +/// +/// `out` must reference at least `OUT_LEN` writable bytes. unsafe fn write_cv_words(out: *mut u8, cv: &[u32; 8]) { for (j, &word) in cv.iter().enumerate() { let bytes = word.to_le_bytes(); // SAFETY: caller guarantees one full CV output is writable at `out`. - unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), out.add(j * 4), 4) }; + unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), out.add(j.strict_mul(4)), 4) }; } } @@ -1866,8 +1954,8 @@ fn rot_lanes_left_3(v: core::simd::u32x4) -> core::simd::u32x4 { fn rotr32(v: core::simd::u32x4) -> core::simd::u32x4 { debug_assert!(N > 0 && N < 32); let s0 = core::simd::u32x4::splat(N); - let s1 = core::simd::u32x4::splat(32 - N); - (v >> s0) | (v << s1) + let s1 = core::simd::u32x4::splat(32u32.strict_sub(N)); + core::ops::BitOr::bitor(core::ops::Shr::shr(v, s0), core::ops::Shl::shl(v, s1)) } #[cfg(any(target_arch = "s390x", target_arch = "powerpc64", target_arch = "riscv64"))] @@ -1919,22 +2007,23 @@ fn compress_simd_leaf( chaining_value[7], ]); let mut row2 = Vec4::from_array([super::IV[0], super::IV[1], super::IV[2], super::IV[3]]); - let mut row3 = Vec4::from_array([counter as u32, (counter >> 32) as u32, block_len, flags]); + let (counter_low, counter_high) = counter_words(counter); + let mut row3 = Vec4::from_array([counter_low, counter_high, block_len, flags]); macro_rules! g { ($mx:expr, $my:expr) => {{ - row0 += row1; - row0 += $mx; + row0 = wrapping_add4(row0, row1); + row0 = wrapping_add4(row0, $mx); row3 ^= row0; row3 = rotr32::<16>(row3); - row2 += row3; + row2 = wrapping_add4(row2, row3); row1 ^= row2; row1 = rotr32::<12>(row1); - row0 += row1; - row0 += $my; + row0 = wrapping_add4(row0, row1); + row0 = wrapping_add4(row0, $my); row3 ^= row0; row3 = rotr32::<8>(row3); - row2 += row3; + row2 = wrapping_add4(row2, row3); row1 ^= row2; row1 = rotr32::<7>(row1); }}; @@ -1999,16 +2088,29 @@ fn load_msg_lanes4_contiguous(base: *const u8, block_offset: usize) -> [core::si // SAFETY: caller provides 4 contiguous full chunks; each block load is in-bounds. let b0 = unsafe { words16_from_le_bytes_64(&*base.add(block_offset).cast::<[u8; BLOCK_LEN]>()) }; // SAFETY: see above. - let b1 = unsafe { words16_from_le_bytes_64(&*base.add(CHUNK_LEN + block_offset).cast::<[u8; BLOCK_LEN]>()) }; + let b1 = + unsafe { words16_from_le_bytes_64(&*base.add(CHUNK_LEN.strict_add(block_offset)).cast::<[u8; BLOCK_LEN]>()) }; // SAFETY: see above. - let b2 = unsafe { words16_from_le_bytes_64(&*base.add(2 * CHUNK_LEN + block_offset).cast::<[u8; BLOCK_LEN]>()) }; + let b2 = unsafe { + words16_from_le_bytes_64( + &*base + .add(2usize.strict_mul(CHUNK_LEN).strict_add(block_offset)) + .cast::<[u8; BLOCK_LEN]>(), + ) + }; // SAFETY: see above. - let b3 = unsafe { words16_from_le_bytes_64(&*base.add(3 * CHUNK_LEN + block_offset).cast::<[u8; BLOCK_LEN]>()) }; + let b3 = unsafe { + words16_from_le_bytes_64( + &*base + .add(3usize.strict_mul(CHUNK_LEN).strict_add(block_offset)) + .cast::<[u8; BLOCK_LEN]>(), + ) + }; let mut out = [core::simd::u32x4::splat(0); 16]; let mut i = 0usize; while i < 16 { out[i] = core::simd::u32x4::from_array([b0[i], b1[i], b2[i], b3[i]]); - i += 1; + i = i.strict_add(1); } out } @@ -2024,18 +2126,18 @@ fn g4_simd( mx: core::simd::u32x4, my: core::simd::u32x4, ) { - v[a] += v[b]; - v[a] += mx; + v[a] = wrapping_add4(v[a], v[b]); + v[a] = wrapping_add4(v[a], mx); v[d] ^= v[a]; v[d] = rotr32::<16>(v[d]); - v[c] += v[d]; + v[c] = wrapping_add4(v[c], v[d]); v[b] ^= v[c]; v[b] = rotr32::<12>(v[b]); - v[a] += v[b]; - v[a] += my; + v[a] = wrapping_add4(v[a], v[b]); + v[a] = wrapping_add4(v[a], my); v[d] ^= v[a]; v[d] = rotr32::<8>(v[d]); - v[c] += v[d]; + v[c] = wrapping_add4(v[c], v[d]); v[b] ^= v[c]; v[b] = rotr32::<7>(v[b]); } @@ -2059,27 +2161,27 @@ fn round4_simd(v: &mut [core::simd::u32x4; 16], m: &[core::simd::u32x4; 16], r: fn load_parent_msg_lanes4(children: &[[u8; OUT_LEN]], base: usize, rem: usize) -> [core::simd::u32x4; 16] { type Vec4 = core::simd::u32x4; debug_assert!(rem > 0 && rem <= 4); - let last = base + rem - 1; + let last = base.strict_add(rem).strict_sub(1); let idx0 = base; - let idx1 = if rem > 1 { base + 1 } else { last }; - let idx2 = if rem > 2 { base + 2 } else { last }; - let idx3 = if rem > 3 { base + 3 } else { last }; - - let l0 = words8_from_le_bytes_32(&children[2 * idx0]); - let r0 = words8_from_le_bytes_32(&children[2 * idx0 + 1]); - let l1 = words8_from_le_bytes_32(&children[2 * idx1]); - let r1 = words8_from_le_bytes_32(&children[2 * idx1 + 1]); - let l2 = words8_from_le_bytes_32(&children[2 * idx2]); - let r2 = words8_from_le_bytes_32(&children[2 * idx2 + 1]); - let l3 = words8_from_le_bytes_32(&children[2 * idx3]); - let r3 = words8_from_le_bytes_32(&children[2 * idx3 + 1]); + let idx1 = if rem > 1 { base.strict_add(1) } else { last }; + let idx2 = if rem > 2 { base.strict_add(2) } else { last }; + let idx3 = if rem > 3 { base.strict_add(3) } else { last }; + + let l0 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx0)]); + let r0 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx0).strict_add(1)]); + let l1 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx1)]); + let r1 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx1).strict_add(1)]); + let l2 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx2)]); + let r2 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx2).strict_add(1)]); + let l3 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx3)]); + let r3 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx3).strict_add(1)]); let mut msg = [Vec4::splat(0); 16]; let mut w = 0usize; while w < 8 { msg[w] = Vec4::from_array([l0[w], l1[w], l2[w], l3[w]]); - msg[w + 8] = Vec4::from_array([r0[w], r1[w], r2[w], r3[w]]); - w += 1; + msg[w.strict_add(8)] = Vec4::from_array([r0[w], r1[w], r2[w], r3[w]]); + w = w.strict_add(1); } msg } @@ -2088,7 +2190,7 @@ fn load_parent_msg_lanes4(children: &[[u8; OUT_LEN]], base: usize, rem: usize) - #[inline(always)] fn parent_cvs_many4_simd(children: &[[u8; OUT_LEN]], key_words: [u32; 8], flags: u32, out: &mut [[u8; OUT_LEN]]) { type Vec4 = core::simd::u32x4; - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if out.is_empty() { return; } @@ -2096,7 +2198,7 @@ fn parent_cvs_many4_simd(children: &[[u8; OUT_LEN]], key_words: [u32; 8], flags: let mut i = 0usize; while i < out.len() { - let rem = core::cmp::min(4, out.len() - i); + let rem = core::cmp::min(4, out.len().strict_sub(i)); let msg = load_parent_msg_lanes4(children, i, rem); let mut v = [ @@ -2114,7 +2216,7 @@ fn parent_cvs_many4_simd(children: &[[u8; OUT_LEN]], key_words: [u32; 8], flags: Vec4::splat(super::IV[3]), Vec4::splat(0), Vec4::splat(0), - Vec4::splat(BLOCK_LEN as u32), + Vec4::splat(BLOCK_LEN_U32), Vec4::splat(parent_flags), ]; @@ -2129,8 +2231,8 @@ fn parent_cvs_many4_simd(children: &[[u8; OUT_LEN]], key_words: [u32; 8], flags: let mut x = [Vec4::splat(0); 8]; let mut j = 0usize; while j < 8 { - x[j] = v[j] ^ v[j + 8]; - j += 1; + x[j] = v[j] ^ v[j.strict_add(8)]; + j = j.strict_add(1); } let xa0 = x[0].to_array(); let xa1 = x[1].to_array(); @@ -2143,17 +2245,23 @@ fn parent_cvs_many4_simd(children: &[[u8; OUT_LEN]], key_words: [u32; 8], flags: let mut lane = 0usize; while lane < rem { - out[i + lane] = super::words8_to_le_bytes(&[ + out[i.strict_add(lane)] = super::words8_to_le_bytes(&[ xa0[lane], xa1[lane], xa2[lane], xa3[lane], xa4[lane], xa5[lane], xa6[lane], xa7[lane], ]); - lane += 1; + lane = lane.strict_add(1); } - i += rem; + i = i.strict_add(rem); } } #[cfg(any(target_arch = "s390x", target_arch = "powerpc64", target_arch = "riscv64"))] #[inline(always)] +/// Hashes four contiguous full chunks with the portable SIMD kernel. +/// +/// # Safety +/// +/// `input` must reference four readable chunks and `out` must reference four +/// writable chaining values. unsafe fn hash4_contiguous_full_chunks_simd(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { type Vec4 = core::simd::u32x4; let mut h = [ @@ -2166,25 +2274,22 @@ unsafe fn hash4_contiguous_full_chunks_simd(input: *const u8, key: &[u32; 8], co Vec4::splat(key[6]), Vec4::splat(key[7]), ]; - let counter_low = Vec4::from_array([ - counter as u32, - counter.wrapping_add(1) as u32, - counter.wrapping_add(2) as u32, - counter.wrapping_add(3) as u32, - ]); - let counter_high = Vec4::from_array([ - (counter >> 32) as u32, - (counter.wrapping_add(1) >> 32) as u32, - (counter.wrapping_add(2) >> 32) as u32, - (counter.wrapping_add(3) >> 32) as u32, - ]); + let [(low0, high0), (low1, high1), (low2, high2), (low3, high3)] = [ + counter, + counter.wrapping_add(1), + counter.wrapping_add(2), + counter.wrapping_add(3), + ] + .map(counter_words); + let counter_low = Vec4::from_array([low0, low1, low2, low3]); + let counter_high = Vec4::from_array([high0, high1, high2, high3]); let mut block_idx = 0usize; - while block_idx < (CHUNK_LEN / BLOCK_LEN) { - let msg = load_msg_lanes4_contiguous(input, block_idx * BLOCK_LEN); + while block_idx < CHUNK_LEN.strict_div(BLOCK_LEN) { + let msg = load_msg_lanes4_contiguous(input, block_idx.strict_mul(BLOCK_LEN)); let block_flags = flags | if block_idx == 0 { CHUNK_START } else { 0 } - | if block_idx + 1 == (CHUNK_LEN / BLOCK_LEN) { + | if block_idx.strict_add(1) == CHUNK_LEN.strict_div(BLOCK_LEN) { super::CHUNK_END } else { 0 @@ -2204,7 +2309,7 @@ unsafe fn hash4_contiguous_full_chunks_simd(input: *const u8, key: &[u32; 8], co Vec4::splat(super::IV[3]), counter_low, counter_high, - Vec4::splat(BLOCK_LEN as u32), + Vec4::splat(BLOCK_LEN_U32), Vec4::splat(block_flags), ]; round4_simd(&mut v, &msg, 0); @@ -2216,10 +2321,10 @@ unsafe fn hash4_contiguous_full_chunks_simd(input: *const u8, key: &[u32; 8], co round4_simd(&mut v, &msg, 6); let mut i = 0usize; while i < 8 { - h[i] = v[i] ^ v[i + 8]; - i += 1; + h[i] = v[i] ^ v[i.strict_add(8)]; + i = i.strict_add(1); } - block_idx += 1; + block_idx = block_idx.strict_add(1); } let h0 = h[0].to_array(); @@ -2234,7 +2339,7 @@ unsafe fn hash4_contiguous_full_chunks_simd(input: *const u8, key: &[u32; 8], co let mut lane = 0usize; while lane < 4 { // SAFETY: caller guarantees `out` is valid for 4 contiguous `OUT_LEN` outputs. - let dst = unsafe { out.add(lane * OUT_LEN) }; + let dst = unsafe { out.add(lane.strict_mul(OUT_LEN)) }; let words = [ h0[lane], h1[lane], h2[lane], h3[lane], h4[lane], h5[lane], h6[lane], h7[lane], ]; @@ -2242,16 +2347,21 @@ unsafe fn hash4_contiguous_full_chunks_simd(input: *const u8, key: &[u32; 8], co while word < 8 { let bytes = words[word].to_le_bytes(); // SAFETY: caller guarantees out is valid for 4 CV outputs. - unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), dst.add(word * 4), 4) }; - word += 1; + unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), dst.add(word.strict_mul(4)), 4) }; + word = word.strict_add(1); } - lane += 1; + lane = lane.strict_add(1); } } // s390x vector wrappers #[cfg(target_arch = "s390x")] +/// Compresses one block with the s390x vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[target_feature(enable = "vector")] unsafe fn compress_s390x_vector( chaining_value: &[u32; 8], @@ -2276,6 +2386,12 @@ fn compress_s390x_vector_wrapper( } #[cfg(target_arch = "s390x")] +/// Writes one root-output block with the s390x vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility, and `out` must reference +/// one writable BLAKE3 output block. #[target_feature(enable = "vector")] unsafe fn root_output_blocks1_s390x_vector( chaining_value: &[u32; 8], @@ -2293,6 +2409,11 @@ unsafe fn root_output_blocks1_s390x_vector( } #[cfg(target_arch = "s390x")] +/// Compresses full chunk blocks with the s390x vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[target_feature(enable = "vector")] unsafe fn chunk_compress_blocks_s390x_vector( chaining_value: &mut [u32; 8], @@ -2323,11 +2444,11 @@ unsafe fn chunk_compress_blocks_s390x_vector( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ) }); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); return; } @@ -2342,11 +2463,11 @@ unsafe fn chunk_compress_blocks_s390x_vector( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ) }); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } @@ -2363,6 +2484,11 @@ fn chunk_compress_blocks_s390x_vector_wrapper( } #[cfg(target_arch = "s390x")] +/// Compresses a parent node with the s390x vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[target_feature(enable = "vector")] unsafe fn parent_cv_s390x_vector( left_child_cv: [u32; 8], @@ -2377,7 +2503,7 @@ unsafe fn parent_cv_s390x_vector( &key_words, &block_words, 0, - BLOCK_LEN as u32, + BLOCK_LEN_U32, PARENT | flags, )) } @@ -2394,6 +2520,12 @@ fn parent_cv_s390x_vector_wrapper( } #[cfg(target_arch = "s390x")] +/// Hashes one full chunk with the s390x vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility, `input` must reference +/// one readable chunk, and `out` must reference one writable chaining value. #[target_feature(enable = "vector")] unsafe fn hash_one_chunk_s390x_vector(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { let mut cv = *key; @@ -2412,12 +2544,19 @@ unsafe fn hash_one_chunk_s390x_vector(input: *const u8, key: &[u32; 8], counter: let start = if blocks_compressed == 0 { CHUNK_START } else { 0 }; let tail_flags = flags | start | super::CHUNK_END; // SAFETY: `compress_s390x_vector` is gated by the current function's vector target feature. - cv = first_8_words(unsafe { compress_s390x_vector(&cv, &tail_words, counter, BLOCK_LEN as u32, tail_flags) }); + cv = first_8_words(unsafe { compress_s390x_vector(&cv, &tail_words, counter, BLOCK_LEN_U32, tail_flags) }); // SAFETY: caller guarantees one full CV output is writable at `out`. unsafe { write_cv_words(out, &cv) }; } #[cfg(target_arch = "s390x")] +/// Hashes contiguous chunks with the s390x vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility, `input` must reference +/// `num_chunks` readable chunks, and `out` must reference `num_chunks` writable +/// chaining values. #[target_feature(enable = "vector")] unsafe fn hash_many_contiguous_s390x_vector( input: *const u8, @@ -2430,36 +2569,43 @@ unsafe fn hash_many_contiguous_s390x_vector( debug_assert!(num_chunks != 0); let mut idx = 0usize; - while idx + 4 <= num_chunks { + while idx.strict_add(4) <= num_chunks { // SAFETY: `idx + 4 <= num_chunks` guarantees four full contiguous chunks and outputs remain. unsafe { hash4_contiguous_full_chunks_simd( - input.add(idx * CHUNK_LEN), + input.add(idx.strict_mul(CHUNK_LEN)), key, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("BLAKE3 chunk index fits in u64")), flags, - out.add(idx * OUT_LEN), + out.add(idx.strict_mul(OUT_LEN)), ); } - idx += 4; + idx = idx.strict_add(4); } while idx < num_chunks { // SAFETY: `idx < num_chunks` guarantees one full chunk/output remains. unsafe { hash_one_chunk_s390x_vector( - input.add(idx * CHUNK_LEN), + input.add(idx.strict_mul(CHUNK_LEN)), key, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("BLAKE3 chunk index fits in u64")), flags, - out.add(idx * OUT_LEN), + out.add(idx.strict_mul(OUT_LEN)), ); } - idx += 1; + idx = idx.strict_add(1); } } #[cfg(target_arch = "s390x")] +/// Dispatches contiguous chunks through the s390x vector implementation. +/// +/// # Safety +/// +/// `input` must reference `num_chunks` readable chunks, `out` must reference +/// `num_chunks` writable chaining values, and the executing CPU must support +/// the vector facility. unsafe fn hash_many_contiguous_s390x_vector_wrapper( input: *const u8, num_chunks: usize, @@ -2476,6 +2622,11 @@ unsafe fn hash_many_contiguous_s390x_vector_wrapper( #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "vsx")] +/// Compresses one block with the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// The current CPU must support VSX. unsafe fn compress_power_vsx( chaining_value: &[u32; 8], block_words: &[u32; 16], @@ -2500,6 +2651,12 @@ fn compress_power_vsx_wrapper( #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "vsx")] +/// Writes one root-output block with the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// The current CPU must support VSX and `out` must reference one writable +/// BLAKE3 output block. unsafe fn root_output_blocks1_power_vsx( chaining_value: &[u32; 8], block_words: &[u32; 16], @@ -2517,6 +2674,11 @@ unsafe fn root_output_blocks1_power_vsx( #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "vsx")] +/// Compresses full chunk blocks with the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// The current CPU must support VSX. unsafe fn chunk_compress_blocks_power_vsx( chaining_value: &mut [u32; 8], chunk_counter: u64, @@ -2524,11 +2686,10 @@ unsafe fn chunk_compress_blocks_power_vsx( blocks_compressed: &mut u8, blocks: &[u8], ) { - debug_assert_eq!(blocks.len() % BLOCK_LEN, 0); + let (block_slices, remainder) = blocks.as_chunks::(); + debug_assert!(remainder.is_empty()); - if blocks.len() == BLOCK_LEN { - // SAFETY: `blocks` is exactly one block, and `[u8; BLOCK_LEN]` has 1-byte alignment. - let block_bytes: &[u8; BLOCK_LEN] = unsafe { &*(blocks.as_ptr().cast()) }; + if let [block_bytes] = block_slices { let start = if *blocks_compressed == 0 { CHUNK_START } else { 0 }; let block_words = words16_from_le_bytes_64(block_bytes); // SAFETY: `compress_power_vsx` is gated by the current function's VSX target feature. @@ -2537,16 +2698,14 @@ unsafe fn chunk_compress_blocks_power_vsx( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ) }); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = (*blocks_compressed).strict_add(1); return; } - let (block_slices, remainder) = blocks.as_chunks::(); - debug_assert!(remainder.is_empty()); for block_bytes in block_slices { let start = if *blocks_compressed == 0 { CHUNK_START } else { 0 }; let block_words = words16_from_le_bytes_64(block_bytes); @@ -2556,11 +2715,11 @@ unsafe fn chunk_compress_blocks_power_vsx( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ) }); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = (*blocks_compressed).strict_add(1); } } @@ -2578,6 +2737,11 @@ fn chunk_compress_blocks_power_vsx_wrapper( #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "vsx")] +/// Compresses a parent node with the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// The current CPU must support VSX. unsafe fn parent_cv_power_vsx( left_child_cv: [u32; 8], right_child_cv: [u32; 8], @@ -2591,7 +2755,7 @@ unsafe fn parent_cv_power_vsx( &key_words, &block_words, 0, - BLOCK_LEN as u32, + BLOCK_LEN_U32, PARENT | flags, )) } @@ -2609,10 +2773,16 @@ fn parent_cv_power_vsx_wrapper( #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "vsx")] +/// Hashes one full chunk with the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// The current CPU must support VSX, `input` must reference one readable +/// chunk, and `out` must reference one writable chaining value. unsafe fn hash_one_chunk_power_vsx(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { let mut cv = *key; let mut blocks_compressed = 0u8; - let body_len = CHUNK_LEN - BLOCK_LEN; + let body_len = CHUNK_LEN.strict_sub(BLOCK_LEN); // SAFETY: caller guarantees one full chunk is readable from `input`. let body = unsafe { core::slice::from_raw_parts(input, body_len) }; // SAFETY: caller upholds the chunk pointer/length precondition. @@ -2626,13 +2796,20 @@ unsafe fn hash_one_chunk_power_vsx(input: *const u8, key: &[u32; 8], counter: u6 let start = if blocks_compressed == 0 { CHUNK_START } else { 0 }; let tail_flags = flags | start | super::CHUNK_END; // SAFETY: `compress_power_vsx` is gated by the current function's VSX target feature. - cv = first_8_words(unsafe { compress_power_vsx(&cv, &tail_words, counter, BLOCK_LEN as u32, tail_flags) }); + cv = first_8_words(unsafe { compress_power_vsx(&cv, &tail_words, counter, BLOCK_LEN_U32, tail_flags) }); // SAFETY: caller guarantees one full CV output is writable at `out`. unsafe { write_cv_words(out, &cv) }; } #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "vsx")] +/// Hashes contiguous chunks with the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// The current CPU must support VSX, `input` must reference `num_chunks` +/// readable chunks, and `out` must reference `num_chunks` writable chaining +/// values. unsafe fn hash_many_contiguous_power_vsx( input: *const u8, num_chunks: usize, @@ -2644,36 +2821,42 @@ unsafe fn hash_many_contiguous_power_vsx( debug_assert!(num_chunks != 0); let mut idx = 0usize; - while idx + 4 <= num_chunks { + while idx.strict_add(4) <= num_chunks { // SAFETY: `idx + 4 <= num_chunks` guarantees four full contiguous chunks and outputs remain. unsafe { hash4_contiguous_full_chunks_simd( - input.add(idx * CHUNK_LEN), + input.add(idx.strict_mul(CHUNK_LEN)), key, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("BLAKE3 chunk index fits in u64")), flags, - out.add(idx * OUT_LEN), + out.add(idx.strict_mul(OUT_LEN)), ); } - idx += 4; + idx = idx.strict_add(4); } while idx < num_chunks { // SAFETY: `idx < num_chunks` guarantees one full chunk/output remains. unsafe { hash_one_chunk_power_vsx( - input.add(idx * CHUNK_LEN), + input.add(idx.strict_mul(CHUNK_LEN)), key, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("BLAKE3 chunk index fits in u64")), flags, - out.add(idx * OUT_LEN), + out.add(idx.strict_mul(OUT_LEN)), ); } - idx += 1; + idx = idx.strict_add(1); } } #[cfg(target_arch = "powerpc64")] +/// Dispatches contiguous chunks through the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// `input` must reference `num_chunks` readable chunks, `out` must reference +/// `num_chunks` writable chaining values, and the current CPU must support VSX. unsafe fn hash_many_contiguous_power_vsx_wrapper( input: *const u8, num_chunks: usize, @@ -2703,7 +2886,7 @@ fn compress_simd_leaf_riscv( let mut w = 0usize; while w < 16 { msg[w] = Vec4::splat(block_words[w]); - w += 1; + w = w.strict_add(1); } let mut v = [ @@ -2719,8 +2902,8 @@ fn compress_simd_leaf_riscv( Vec4::splat(super::IV[1]), Vec4::splat(super::IV[2]), Vec4::splat(super::IV[3]), - Vec4::splat(counter as u32), - Vec4::splat((counter >> 32) as u32), + Vec4::splat(counter_words(counter).0), + Vec4::splat(counter_words(counter).1), Vec4::splat(block_len), Vec4::splat(flags), ]; @@ -2756,6 +2939,11 @@ fn compress_simd_leaf_riscv( } #[cfg(target_arch = "riscv64")] +/// Compresses one block with the RISC-V vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[target_feature(enable = "v")] unsafe fn compress_riscv_v( chaining_value: &[u32; 8], @@ -2780,6 +2968,12 @@ fn compress_riscv_v_wrapper( } #[cfg(target_arch = "riscv64")] +/// Writes one root-output block with the RISC-V vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension, and `out` must +/// reference one writable BLAKE3 output block. #[target_feature(enable = "v")] unsafe fn root_output_blocks1_riscv_v( chaining_value: &[u32; 8], @@ -2797,6 +2991,11 @@ unsafe fn root_output_blocks1_riscv_v( } #[cfg(target_arch = "riscv64")] +/// Compresses full chunk blocks with the RISC-V vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[target_feature(enable = "v")] unsafe fn chunk_compress_blocks_riscv_v( chaining_value: &mut [u32; 8], @@ -2818,11 +3017,11 @@ unsafe fn chunk_compress_blocks_riscv_v( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ) }); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); return; } @@ -2837,11 +3036,11 @@ unsafe fn chunk_compress_blocks_riscv_v( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ) }); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } @@ -2858,6 +3057,11 @@ fn chunk_compress_blocks_riscv_v_wrapper( } #[cfg(target_arch = "riscv64")] +/// Compresses a parent node with the RISC-V vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[target_feature(enable = "v")] unsafe fn parent_cv_riscv_v( left_child_cv: [u32; 8], @@ -2872,7 +3076,7 @@ unsafe fn parent_cv_riscv_v( &key_words, &block_words, 0, - BLOCK_LEN as u32, + BLOCK_LEN_U32, PARENT | flags, )) } @@ -2889,11 +3093,17 @@ fn parent_cv_riscv_v_wrapper( } #[cfg(target_arch = "riscv64")] +/// Hashes one full chunk with the RISC-V vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension, `input` must reference +/// one readable chunk, and `out` must reference one writable chaining value. #[target_feature(enable = "v")] unsafe fn hash_one_chunk_riscv_v(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { let mut cv = *key; let mut blocks_compressed = 0u8; - let body_len = CHUNK_LEN - BLOCK_LEN; + let body_len = CHUNK_LEN.strict_sub(BLOCK_LEN); // SAFETY: caller guarantees one full chunk is readable from `input`. let body = unsafe { core::slice::from_raw_parts(input, body_len) }; // SAFETY: caller upholds the chunk pointer/length precondition. @@ -2907,12 +3117,19 @@ unsafe fn hash_one_chunk_riscv_v(input: *const u8, key: &[u32; 8], counter: u64, let start = if blocks_compressed == 0 { CHUNK_START } else { 0 }; let tail_flags = flags | start | super::CHUNK_END; // SAFETY: this kernel is only dispatched when `riscv::V` is available. - cv = first_8_words(unsafe { compress_riscv_v(&cv, &tail_words, counter, BLOCK_LEN as u32, tail_flags) }); + cv = first_8_words(unsafe { compress_riscv_v(&cv, &tail_words, counter, BLOCK_LEN_U32, tail_flags) }); // SAFETY: caller guarantees one full CV output is writable at `out`. unsafe { write_cv_words(out, &cv) }; } #[cfg(target_arch = "riscv64")] +/// Hashes contiguous chunks with the RISC-V vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension, `input` must reference +/// `num_chunks` readable chunks, and `out` must reference `num_chunks` writable +/// chaining values. #[target_feature(enable = "v")] unsafe fn hash_many_contiguous_riscv_v( input: *const u8, @@ -2925,36 +3142,43 @@ unsafe fn hash_many_contiguous_riscv_v( debug_assert!(num_chunks != 0); let mut idx = 0usize; - while idx + 4 <= num_chunks { + while idx.strict_add(4) <= num_chunks { // SAFETY: `idx + 4 <= num_chunks` guarantees four full contiguous chunks and outputs remain. unsafe { hash4_contiguous_full_chunks_simd( - input.add(idx * CHUNK_LEN), + input.add(idx.strict_mul(CHUNK_LEN)), key, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("BLAKE3 chunk index fits in u64")), flags, - out.add(idx * OUT_LEN), + out.add(idx.strict_mul(OUT_LEN)), ); } - idx += 4; + idx = idx.strict_add(4); } while idx < num_chunks { // SAFETY: `idx < num_chunks` guarantees one full chunk/output remains. unsafe { hash_one_chunk_riscv_v( - input.add(idx * CHUNK_LEN), + input.add(idx.strict_mul(CHUNK_LEN)), key, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("BLAKE3 chunk index fits in u64")), flags, - out.add(idx * OUT_LEN), + out.add(idx.strict_mul(OUT_LEN)), ); } - idx += 1; + idx = idx.strict_add(1); } } #[cfg(target_arch = "riscv64")] +/// Dispatches contiguous chunks through the RISC-V vector implementation. +/// +/// # Safety +/// +/// `input` must reference `num_chunks` readable chunks, `out` must reference +/// `num_chunks` writable chaining values, and the executing CPU must support +/// the RISC-V V extension. unsafe fn hash_many_contiguous_riscv_v_wrapper( input: *const u8, num_chunks: usize, @@ -2983,6 +3207,11 @@ fn x86_compress_cv_portable_wrapper( } #[cfg(target_arch = "x86_64")] +/// Compress one block with the SSE4.1 kernel. +/// +/// # Safety +/// +/// SSE4.1 must be available and `block` must be readable for 64 bytes. unsafe fn x86_compress_cv_sse41_wrapper( cv: &[u32; 8], block: *const u8, @@ -2996,6 +3225,11 @@ unsafe fn x86_compress_cv_sse41_wrapper( } #[cfg(target_arch = "x86_64")] +/// Compress one block with the AVX2 kernel. +/// +/// # Safety +/// +/// AVX2 must be available and `block` must be readable for 64 bytes. unsafe fn x86_compress_cv_avx2_wrapper( cv: &[u32; 8], block: *const u8, @@ -3009,6 +3243,12 @@ unsafe fn x86_compress_cv_avx2_wrapper( } #[cfg(target_arch = "x86_64")] +/// Compress one block with the AVX-512 kernel. +/// +/// # Safety +/// +/// The AVX-512 dispatch requirements must hold and `block` must be readable +/// for 64 bytes. unsafe fn x86_compress_cv_avx512_wrapper( cv: &[u32; 8], block: *const u8, @@ -3031,6 +3271,12 @@ unsafe fn x86_compress_cv_avx512_wrapper( } #[cfg(all(feature = "diag", target_arch = "x86_64"))] +/// Compress one block with the owned AVX-512 kernel. +/// +/// # Safety +/// +/// The owned AVX-512 dispatch requirements must hold and `block` must be +/// readable for 64 bytes. unsafe fn x86_compress_cv_avx512_owned_wrapper( cv: &[u32; 8], block: *const u8, @@ -3129,11 +3375,11 @@ fn chunk_compress_blocks_avx512_owned_wrapper( chaining_value, block_bytes.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); } - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } @@ -3186,6 +3432,12 @@ fn parent_cv_sse41_wrapper( } #[cfg(target_arch = "x86_64")] +/// Hash contiguous chunks with the SSE4.1 kernel. +/// +/// # Safety +/// +/// SSE4.1 must be available. `input` and `out` must cover `num_chunks` full +/// chunks and chaining values, respectively, without overlapping. unsafe fn hash_many_contiguous_sse41_wrapper( input: *const u8, mut num_chunks: usize, @@ -3210,22 +3462,22 @@ unsafe fn hash_many_contiguous_sse41_wrapper( // SAFETY: dispatch selects this kernel only when SSE4.1 is available; the // caller guarantees `input`/`out` cover the full `num_chunks` buffer. unsafe { - super::x86_64::sse41::hash4( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::sse41::hash4(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, out, - ); - input = input.add(super::x86_64::sse41::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::sse41::DEGREE * OUT_LEN); + }); + input = input.add(super::x86_64::sse41::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::sse41::DEGREE.strict_mul(OUT_LEN)); } counter = counter.wrapping_add(super::x86_64::sse41::DEGREE as u64); - num_chunks -= super::x86_64::sse41::DEGREE; + num_chunks = num_chunks.strict_sub(super::x86_64::sse41::DEGREE); } if num_chunks != 0 { @@ -3233,7 +3485,7 @@ unsafe fn hash_many_contiguous_sse41_wrapper( // the final chunk pointer into unused lanes and only copy the needed // outputs. // SAFETY: `num_chunks != 0`, and `input` is valid for `num_chunks * CHUNK_LEN` bytes. - let last = unsafe { input.add((num_chunks - 1) * CHUNK_LEN) }; + let last = unsafe { input.add(num_chunks.strict_sub(1).strict_mul(CHUNK_LEN)) }; // SAFETY: all pointers are within the caller-provided `input` buffer. let ptrs = unsafe { [ @@ -3248,23 +3500,26 @@ unsafe fn hash_many_contiguous_sse41_wrapper( // SAFETY: SSE4.1 is available for this wrapper, `ptrs` are in-bounds for // full chunks, and `tmp`/`out` are large enough for the copied outputs. unsafe { - super::x86_64::sse41::hash4( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::sse41::hash4(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, - tmp.as_mut_ptr(), - ); - core::ptr::copy_nonoverlapping(tmp.as_ptr(), out, num_chunks * OUT_LEN); + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, + out: tmp.as_mut_ptr(), + }); + core::ptr::copy_nonoverlapping(tmp.as_ptr(), out, num_chunks.strict_mul(OUT_LEN)); } } } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows") +))] /// Hash a sub-degree contiguous AVX2 chunk tail by duplicating the final lane. /// /// # Safety @@ -3288,7 +3543,7 @@ unsafe fn hash_many_avx2_owned_duplicate_tail( // SAFETY: Computing the final real chunk pointer because: // 1. The caller guarantees `num_chunks` is non-zero. // 2. The caller guarantees `input` is readable for `num_chunks * CHUNK_LEN` bytes. - let last = unsafe { input.add((num_chunks - 1) * CHUNK_LEN) }; + let last = unsafe { input.add(num_chunks.strict_sub(1).strict_mul(CHUNK_LEN)) }; // SAFETY: Constructing a full 8-lane batch because: // 1. Lanes `< num_chunks` point at distinct real chunks in the caller-provided input. // 2. Extra lanes duplicate `last`, which is valid for one full chunk. @@ -3312,23 +3567,32 @@ unsafe fn hash_many_avx2_owned_duplicate_tail( // 2. `ptrs` is a valid full 8-lane batch and `tmp` holds all lane outputs. // 3. `out` is writable for the first `num_chunks * OUT_LEN` bytes. unsafe { - super::x86_64::avx2::hash8_owned( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::avx2::hash8_owned(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, - tmp.as_mut_ptr(), - ); - core::ptr::copy_nonoverlapping(tmp.as_ptr(), out, num_chunks * OUT_LEN); + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, + out: tmp.as_mut_ptr(), + }); + core::ptr::copy_nonoverlapping(tmp.as_ptr(), out, num_chunks.strict_mul(OUT_LEN)); } } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows") +))] #[inline(always)] +/// Hash one full chunk through the serial AVX2 compressor. +/// +/// # Safety +/// +/// AVX2 must be available. `input` must be readable for one full chunk, and +/// `out` must be writable for one chaining value without overlapping `input`. unsafe fn hash_one_chunk_avx2_owned_serial(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { let mut cv = *key; for block_idx in 0..(CHUNK_LEN / BLOCK_LEN) { @@ -3336,7 +3600,7 @@ unsafe fn hash_one_chunk_avx2_owned_serial(input: *const u8, key: &[u32; 8], cou if block_idx == 0 { block_flags |= CHUNK_START; } - if block_idx + 1 == CHUNK_LEN / BLOCK_LEN { + if block_idx.strict_add(1) == CHUNK_LEN / BLOCK_LEN { block_flags |= super::CHUNK_END; } @@ -3348,9 +3612,9 @@ unsafe fn hash_one_chunk_avx2_owned_serial(input: *const u8, key: &[u32; 8], cou cv = unsafe { super::x86_64::compress_cv_avx2_bytes( &cv, - input.add(block_idx * BLOCK_LEN), + input.add(block_idx.strict_mul(BLOCK_LEN)), counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, block_flags, ) }; @@ -3364,6 +3628,12 @@ unsafe fn hash_one_chunk_avx2_owned_serial(input: *const u8, key: &[u32; 8], cou } #[cfg(target_arch = "x86_64")] +/// Hash contiguous chunks with the production AVX2 routing policy. +/// +/// # Safety +/// +/// AVX2 must be available. `input` and `out` must cover `num_chunks` full +/// chunks and chaining values, respectively, without overlapping. unsafe fn hash_many_contiguous_avx2_inner( input: *const u8, mut num_chunks: usize, @@ -3396,19 +3666,19 @@ unsafe fn hash_many_contiguous_avx2_inner( // 3. `out` is valid for `DEGREE * OUT_LEN` bytes. // 4. Sub-degree tails are handled below by the explicit per-degree policy. unsafe { - super::x86_64::avx2::hash8_owned( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::avx2::hash8_owned(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, out, - ); - input = input.add(super::x86_64::avx2::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::avx2::DEGREE * OUT_LEN); + }); + input = input.add(super::x86_64::avx2::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::avx2::DEGREE.strict_mul(OUT_LEN)); } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] @@ -3417,29 +3687,28 @@ unsafe fn hash_many_contiguous_avx2_inner( // caller guarantees `input`/`out` cover the full `num_chunks` buffer. // SAFETY: AVX2 is available for this wrapper, `ptrs` are in-bounds for // full chunks, and `out` is large enough for `DEGREE * OUT_LEN` bytes. - super::x86_64::avx2::hash8( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::avx2::hash8(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, out, - ); - input = input.add(super::x86_64::avx2::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::avx2::DEGREE * OUT_LEN); + }); + input = input.add(super::x86_64::avx2::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::avx2::DEGREE.strict_mul(OUT_LEN)); } counter = counter.wrapping_add(super::x86_64::avx2::DEGREE as u64); - num_chunks -= super::x86_64::avx2::DEGREE; + num_chunks = num_chunks.strict_sub(super::x86_64::avx2::DEGREE); } if num_chunks != 0 { #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] { debug_assert!(num_chunks < super::x86_64::avx2::DEGREE); - debug_assert!(flags <= u8::MAX as u32); if num_chunks == 1 { // SAFETY: Use the owned AVX2 one-chunk tail because: // 1. This wrapper is only selected after AVX2 dispatch. @@ -3479,32 +3748,28 @@ unsafe fn hash_many_contiguous_avx2_inner( // Use the upstream-grade AVX2 asm `hash_many` backend for the remaining // sub-degree tails. Passing `num_inputs = num_chunks` avoids wasting lanes // for the four-chunk shape assigned to assembly by the tail policy. - // SAFETY: This wrapper is only selected when AVX2 is available (checked - // by dispatch). `input` is valid for `num_chunks * CHUNK_LEN` bytes, so - // each `input.add(i * CHUNK_LEN)` stays in-bounds. `out` is valid for - // `num_chunks * OUT_LEN` bytes. let mut ptrs = [input; super::x86_64::avx2::DEGREE]; for (i, ptr) in ptrs.iter_mut().enumerate().take(num_chunks) { // SAFETY: `i < num_chunks` and the caller guarantees `input` is valid // for `num_chunks * CHUNK_LEN` bytes. - *ptr = unsafe { input.add(i * CHUNK_LEN) }; + *ptr = unsafe { input.add(i.strict_mul(CHUNK_LEN)) }; } // SAFETY: AVX2 is available for this kernel per dispatch. `ptrs` points // to `num_chunks` valid chunk inputs, and `out` is valid for // `num_chunks * OUT_LEN` bytes. unsafe { - super::x86_64::asm::hash_many_avx2( - ptrs.as_ptr(), - num_chunks, - CHUNK_LEN / BLOCK_LEN, - key.as_ptr(), + super::x86_64::asm::hash_many_avx2(super::x86_64::asm::HashManyRequest { + inputs: ptrs.as_ptr(), + num_inputs: num_chunks, + blocks: CHUNK_LEN / BLOCK_LEN, + key: key.as_ptr(), counter, - true, - flags as u8, - CHUNK_START as u8, - super::CHUNK_END as u8, + increment_counter: true, + flags: assembly_flags(flags), + flags_start: assembly_flags(CHUNK_START), + flags_end: assembly_flags(super::CHUNK_END), out, - ); + }); } } @@ -3513,7 +3778,7 @@ unsafe fn hash_many_contiguous_avx2_inner( // Non-Linux fallback: hash an 8-lane batch with duplicated final pointers // and copy only the needed outputs. // SAFETY: `num_chunks != 0`, and `input` is valid for `num_chunks * CHUNK_LEN` bytes. - let last = unsafe { input.add((num_chunks - 1) * CHUNK_LEN) }; + let last = unsafe { input.add(num_chunks.strict_sub(1).strict_mul(CHUNK_LEN)) }; // SAFETY: all pointers are within the caller-provided `input` buffer. let ptrs = unsafe { [ @@ -3532,24 +3797,30 @@ unsafe fn hash_many_contiguous_avx2_inner( // SAFETY: AVX2 is available for this wrapper, `ptrs` are in-bounds for // full chunks, and `tmp`/`out` are large enough for the copied outputs. unsafe { - super::x86_64::avx2::hash8( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::avx2::hash8(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, - tmp.as_mut_ptr(), - ); - core::ptr::copy_nonoverlapping(tmp.as_ptr(), out, num_chunks * OUT_LEN); + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, + out: tmp.as_mut_ptr(), + }); + core::ptr::copy_nonoverlapping(tmp.as_ptr(), out, num_chunks.strict_mul(OUT_LEN)); } } } } #[cfg(target_arch = "x86_64")] +/// Forward contiguous chunks into the production AVX2 implementation. +/// +/// # Safety +/// +/// AVX2 must be available. `input` and `out` must satisfy +/// [`HashManyContiguousFn`]'s buffer and non-overlap contract. unsafe fn hash_many_contiguous_avx2_wrapper( input: *const u8, num_chunks: usize, @@ -3564,7 +3835,10 @@ unsafe fn hash_many_contiguous_avx2_wrapper( unsafe { hash_many_contiguous_avx2_inner(input, num_chunks, key, counter, flags, out) }; } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows") +))] #[inline] fn avx512_owned_hash_many_available() -> bool { crate::platform::caps().has( @@ -3577,7 +3851,17 @@ fn avx512_owned_hash_many_available() -> bool { ) } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows") +))] +/// Hash a sub-degree contiguous AVX-512 chunk tail with duplicate lanes. +/// +/// # Safety +/// +/// The owned AVX-512 dispatch requirements must hold. `num_chunks` must be in +/// `1..avx512::DEGREE`, and `input`/`out` must cover that many chunks/CVs +/// without overlapping. unsafe fn hash_many_avx512_owned_duplicate_tail( input: *const u8, num_chunks: usize, @@ -3598,14 +3882,14 @@ unsafe fn hash_many_avx512_owned_duplicate_tail( // 3. Source and destination ranges are disjoint allocations. unsafe { core::ptr::copy_nonoverlapping( - input.add(i * CHUNK_LEN), - tmp_input.as_mut_ptr().add(i * CHUNK_LEN), + input.add(i.strict_mul(CHUNK_LEN)), + tmp_input.as_mut_ptr().add(i.strict_mul(CHUNK_LEN)), CHUNK_LEN, ); } } - let last_src_offset = (num_chunks - 1) * CHUNK_LEN; + let last_src_offset = num_chunks.strict_sub(1).strict_mul(CHUNK_LEN); for i in num_chunks..super::x86_64::avx512::DEGREE { // SAFETY: Duplicating the final real tail chunk because: // 1. `num_chunks != 0`, so `last_src_offset` names an initialized lane. @@ -3614,7 +3898,7 @@ unsafe fn hash_many_avx512_owned_duplicate_tail( unsafe { core::ptr::copy_nonoverlapping( tmp_input.as_ptr().add(last_src_offset), - tmp_input.as_mut_ptr().add(i * CHUNK_LEN), + tmp_input.as_mut_ptr().add(i.strict_mul(CHUNK_LEN)), CHUNK_LEN, ); } @@ -3626,11 +3910,18 @@ unsafe fn hash_many_avx512_owned_duplicate_tail( // 3. `out` is writable for the first `num_chunks * OUT_LEN` bytes. unsafe { super::x86_64::avx512::hash16_contiguous_owned(tmp_input.as_ptr(), key, counter, flags, tmp_out.as_mut_ptr()); - core::ptr::copy_nonoverlapping(tmp_out.as_ptr(), out, num_chunks * OUT_LEN); + core::ptr::copy_nonoverlapping(tmp_out.as_ptr(), out, num_chunks.strict_mul(OUT_LEN)); } } #[cfg(target_arch = "x86_64")] +/// Hash contiguous chunks with the production AVX-512 routing policy. +/// +/// # Safety +/// +/// The AVX-512 dispatch requirements must hold. `input` and `out` must cover +/// `num_chunks` full chunks and chaining values, respectively, without +/// overlapping. unsafe fn hash_many_contiguous_avx512_wrapper( input: *const u8, mut num_chunks: usize, @@ -3646,7 +3937,6 @@ unsafe fn hash_many_contiguous_avx512_wrapper( // by dispatch), the constructed `ptrs` all stay in-bounds for full chunks, // and `out` is valid for `DEGREE * OUT_LEN` bytes. unsafe { - debug_assert!(flags <= u8::MAX as u32); let ptrs = [ input, input.add(CHUNK_LEN), @@ -3665,20 +3955,20 @@ unsafe fn hash_many_contiguous_avx512_wrapper( input.add(14 * CHUNK_LEN), input.add(15 * CHUNK_LEN), ]; - super::x86_64::asm::hash_many_avx512( - ptrs.as_ptr(), - super::x86_64::avx512::DEGREE, - CHUNK_LEN / BLOCK_LEN, - key.as_ptr(), + super::x86_64::asm::hash_many_avx512(super::x86_64::asm::HashManyRequest { + inputs: ptrs.as_ptr(), + num_inputs: super::x86_64::avx512::DEGREE, + blocks: CHUNK_LEN / BLOCK_LEN, + key: key.as_ptr(), counter, - true, - flags as u8, - CHUNK_START as u8, - super::CHUNK_END as u8, + increment_counter: true, + flags: assembly_flags(flags), + flags_start: assembly_flags(CHUNK_START), + flags_end: assembly_flags(super::CHUNK_END), out, - ); - input = input.add(super::x86_64::avx512::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::avx512::DEGREE * OUT_LEN); + }); + input = input.add(super::x86_64::avx512::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::avx512::DEGREE.strict_mul(OUT_LEN)); } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] @@ -3686,18 +3976,17 @@ unsafe fn hash_many_contiguous_avx512_wrapper( // SAFETY: dispatch selects this kernel only when AVX-512 is available; // the caller guarantees `input`/`out` cover the full `num_chunks` buffer. super::x86_64::avx512::hash16_contiguous(input, key, counter, flags, out); - input = input.add(super::x86_64::avx512::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::avx512::DEGREE * OUT_LEN); + input = input.add(super::x86_64::avx512::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::avx512::DEGREE.strict_mul(OUT_LEN)); } counter = counter.wrapping_add(super::x86_64::avx512::DEGREE as u64); - num_chunks -= super::x86_64::avx512::DEGREE; + num_chunks = num_chunks.strict_sub(super::x86_64::avx512::DEGREE); } if num_chunks != 0 { #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] { debug_assert!(num_chunks < super::x86_64::avx512::DEGREE); - debug_assert!(flags <= u8::MAX as u32); // Use the AVX-512 asm `hash_many` backend for the sub-degree tail // (1–15 chunks). The assembly performs its internal cascade with lane // masking. @@ -3716,30 +4005,28 @@ unsafe fn hash_many_contiguous_avx512_wrapper( return; } - // SAFETY: This wrapper is only selected when the AVX-512 kernel is - // available per dispatch. `input`/`out` cover `num_chunks` full chunks. let mut ptrs = [input; super::x86_64::avx512::DEGREE]; for (i, ptr) in ptrs.iter_mut().enumerate().take(num_chunks) { // SAFETY: `i < num_chunks` and the caller guarantees `input` is valid // for `num_chunks * CHUNK_LEN` bytes. - *ptr = unsafe { input.add(i * CHUNK_LEN) }; + *ptr = unsafe { input.add(i.strict_mul(CHUNK_LEN)) }; } // SAFETY: AVX-512 is available for this kernel per dispatch. `ptrs` // points to `num_chunks` valid chunk inputs, and `out` is valid for // `num_chunks * OUT_LEN` bytes. unsafe { - super::x86_64::asm::hash_many_avx512( - ptrs.as_ptr(), - num_chunks, - CHUNK_LEN / BLOCK_LEN, - key.as_ptr(), + super::x86_64::asm::hash_many_avx512(super::x86_64::asm::HashManyRequest { + inputs: ptrs.as_ptr(), + num_inputs: num_chunks, + blocks: CHUNK_LEN / BLOCK_LEN, + key: key.as_ptr(), counter, - true, - flags as u8, - CHUNK_START as u8, - super::CHUNK_END as u8, + increment_counter: true, + flags: assembly_flags(flags), + flags_start: assembly_flags(CHUNK_START), + flags_end: assembly_flags(super::CHUNK_END), out, - ); + }); } } @@ -3756,21 +4043,21 @@ unsafe fn hash_many_contiguous_avx512_wrapper( // `DEGREE * CHUNK_LEN`, so each destination lane is in-bounds. unsafe { core::ptr::copy_nonoverlapping( - input.add(i * CHUNK_LEN), - tmp_input.as_mut_ptr().add(i * CHUNK_LEN), + input.add(i.strict_mul(CHUNK_LEN)), + tmp_input.as_mut_ptr().add(i.strict_mul(CHUNK_LEN)), CHUNK_LEN, ); } } - let last_src_offset = (num_chunks - 1) * CHUNK_LEN; + let last_src_offset = num_chunks.strict_sub(1).strict_mul(CHUNK_LEN); for i in num_chunks..super::x86_64::avx512::DEGREE { // SAFETY: `last_src_offset` points to a previously materialized lane in // `tmp_input`, and destination lane `i` is within the fixed-size buffer. unsafe { core::ptr::copy_nonoverlapping( tmp_input.as_ptr().add(last_src_offset), - tmp_input.as_mut_ptr().add(i * CHUNK_LEN), + tmp_input.as_mut_ptr().add(i.strict_mul(CHUNK_LEN)), CHUNK_LEN, ); } @@ -3780,13 +4067,19 @@ unsafe fn hash_many_contiguous_avx512_wrapper( // valid for the full 16-lane contiguous contract. unsafe { super::x86_64::avx512::hash16_contiguous(tmp_input.as_ptr(), key, counter, flags, tmp_out.as_mut_ptr()); - core::ptr::copy_nonoverlapping(tmp_out.as_ptr(), out, num_chunks * OUT_LEN); + core::ptr::copy_nonoverlapping(tmp_out.as_ptr(), out, num_chunks.strict_mul(OUT_LEN)); } } } } #[cfg(all(feature = "diag", target_arch = "x86_64"))] +/// Hash contiguous chunks with the owned AVX2 diagnostic kernel. +/// +/// # Safety +/// +/// The owned AVX2 dispatch requirements must hold. `input` and `out` must +/// cover `num_chunks` full chunks and chaining values without overlapping. unsafe fn hash_many_contiguous_avx2_owned_wrapper( input: *const u8, mut num_chunks: usize, @@ -3819,22 +4112,22 @@ unsafe fn hash_many_contiguous_avx2_owned_wrapper( // 2. `ptrs` cover 8 full chunks and `out` covers 8 output CVs. // 3. Pointer advancement stays within the caller-provided input/output ranges. unsafe { - super::x86_64::avx2::hash8_owned( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::avx2::hash8_owned(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, out, - ); - input = input.add(super::x86_64::avx2::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::avx2::DEGREE * OUT_LEN); + }); + input = input.add(super::x86_64::avx2::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::avx2::DEGREE.strict_mul(OUT_LEN)); } counter = counter.wrapping_add(super::x86_64::avx2::DEGREE as u64); - num_chunks -= super::x86_64::avx2::DEGREE; + num_chunks = num_chunks.strict_sub(super::x86_64::avx2::DEGREE); } if num_chunks != 0 { @@ -3848,6 +4141,12 @@ unsafe fn hash_many_contiguous_avx2_owned_wrapper( } #[cfg(all(feature = "diag", target_arch = "x86_64"))] +/// Hash contiguous chunks with the owned AVX-512 diagnostic kernel. +/// +/// # Safety +/// +/// The owned AVX-512 dispatch requirements must hold. `input` and `out` must +/// cover `num_chunks` full chunks and chaining values without overlapping. unsafe fn hash_many_contiguous_avx512_owned_wrapper( input: *const u8, mut num_chunks: usize, @@ -3864,11 +4163,11 @@ unsafe fn hash_many_contiguous_avx512_owned_wrapper( // 3. `out` is writable for `DEGREE * OUT_LEN` bytes. unsafe { super::x86_64::avx512::hash16_contiguous_owned(input, key, counter, flags, out); - input = input.add(super::x86_64::avx512::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::avx512::DEGREE * OUT_LEN); + input = input.add(super::x86_64::avx512::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::avx512::DEGREE.strict_mul(OUT_LEN)); } counter = counter.wrapping_add(super::x86_64::avx512::DEGREE as u64); - num_chunks -= super::x86_64::avx512::DEGREE; + num_chunks = num_chunks.strict_sub(super::x86_64::avx512::DEGREE); } if num_chunks != 0 { @@ -3920,6 +4219,13 @@ fn parent_cv_neon_wrapper( } #[cfg(target_arch = "aarch64")] +/// Dispatches contiguous chunks through the AArch64 NEON implementation. +/// +/// # Safety +/// +/// `input` must reference `num_chunks` readable chunks, `out` must reference +/// `num_chunks` writable chaining values, and the current CPU must support +/// NEON. unsafe fn hash_many_contiguous_neon_wrapper( input: *const u8, num_chunks: usize, diff --git a/src/hashes/crypto/blake3/mod.rs b/src/hashes/crypto/blake3/mod.rs index e2844434..7224d34c 100644 --- a/src/hashes/crypto/blake3/mod.rs +++ b/src/hashes/crypto/blake3/mod.rs @@ -2,16 +2,10 @@ //! //! This is a portable, dependency-free implementation suitable for `no_std`. -#![cfg_attr(not(test), deny(clippy::unwrap_used))] -#![cfg_attr(not(test), deny(clippy::expect_used))] -#![allow(clippy::indexing_slicing)] // Audited fixed-size parsing + perf-critical inner loops. - #[cfg(feature = "std")] use core::cell::RefCell; #[cfg(any(feature = "parallel", not(target_endian = "little")))] use core::slice; -#[cfg(all(feature = "parallel", test))] -use core::sync::atomic::{AtomicBool, Ordering}; use core::{cmp::min, mem::MaybeUninit, ptr}; #[cfg(feature = "std")] use std::thread_local; @@ -37,6 +31,10 @@ use self::{control::ParallelPolicyKind, kernels::Kernel}; const OUT_LEN: usize = 32; const KEY_LEN: usize = 32; const BLOCK_LEN: usize = 64; +#[cfg(target_arch = "x86_64")] +const BLOCK_LEN_U8: u8 = 64; +#[cfg(target_arch = "x86_64")] +const BLOCK_LEN_U32: u32 = 64; const CHUNK_LEN: usize = 1024; const OUTPUT_BLOCK_LEN: usize = 2 * OUT_LEN; // Max CV stack depth for incremental hashing. @@ -215,9 +213,8 @@ fn thread_range(thread_index: usize, threads_total: usize, total: usize) -> (usi #[cfg(feature = "parallel")] #[inline] -fn run_parallel_task(task: impl FnOnce()) -> bool { +fn run_parallel_task(task: impl FnOnce()) { task(); - true } #[cfg(feature = "parallel")] @@ -282,12 +279,12 @@ fn reduce_power_of_two_cvs_in_place( let mut cur_is_0 = true; while cur_len > 1 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); debug_assert!(pairs != 0); if cur_is_0 { kernels::parent_cvs_many_from_cvs_inline( kernel.id, - &scratch0[..2 * pairs], + &scratch0[..2usize.strict_mul(pairs)], key_words, flags, &mut scratch1[..pairs], @@ -296,7 +293,7 @@ fn reduce_power_of_two_cvs_in_place( } else { kernels::parent_cvs_many_from_cvs_inline( kernel.id, - &scratch1[..2 * pairs], + &scratch1[..2usize.strict_mul(pairs)], key_words, flags, &mut scratch0[..pairs], @@ -319,7 +316,7 @@ fn hash_full_chunks_cvs_serial( input: &[u8], out: &mut [[u32; 8]], ) { - debug_assert_eq!(input.len(), out.len() * CHUNK_LEN); + debug_assert_eq!(input.len(), out.len().strict_mul(CHUNK_LEN)); const MAX_SIMD_DEGREE: usize = 16; @@ -330,7 +327,7 @@ fn hash_full_chunks_cvs_serial( #[cfg(target_endian = "little")] { while written < out.len() { - let remaining = out.len() - written; + let remaining = out.len().strict_sub(written); let batch = remaining.min(MAX_SIMD_DEGREE); // SAFETY: @@ -342,11 +339,11 @@ fn hash_full_chunks_cvs_serial( (kernel.hash_many_contiguous)(input_ptr, batch, &key_words, chunk_counter, flags, out_ptr); } - written += batch; + written = written.strict_add(batch); chunk_counter = chunk_counter.wrapping_add(batch as u64); // SAFETY: advancing within `input` by whole chunks. unsafe { - input_ptr = input_ptr.add(batch * CHUNK_LEN); + input_ptr = input_ptr.add(batch.strict_mul(CHUNK_LEN)); } } } @@ -356,7 +353,7 @@ fn hash_full_chunks_cvs_serial( #[cfg(not(target_endian = "little"))] while written < out.len() { - let remaining = out.len() - written; + let remaining = out.len().strict_sub(written); let batch = remaining.min(MAX_SIMD_DEGREE); // SAFETY: @@ -366,18 +363,17 @@ fn hash_full_chunks_cvs_serial( (kernel.hash_many_contiguous)(input_ptr, batch, &key_words, chunk_counter, flags, out_buf.as_mut_ptr()); } - for i in 0..batch { - let offset = i * OUT_LEN; - // SAFETY: `out_buf` is `OUT_LEN * MAX_SIMD_DEGREE` bytes, and `i < batch <= MAX_SIMD_DEGREE`. - let cv = unsafe { words8_from_le_bytes_32(&*(out_buf.as_ptr().add(offset) as *const [u8; OUT_LEN])) }; - out[written + i] = cv; + let (out_cvs, remainder) = out_buf.as_chunks::(); + debug_assert!(remainder.is_empty()); + for (i, cv_bytes) in out_cvs[..batch].iter().enumerate() { + out[written.strict_add(i)] = words8_from_le_bytes_32(cv_bytes); } - written += batch; + written = written.strict_add(batch); chunk_counter = chunk_counter.wrapping_add(batch as u64); // SAFETY: advancing within `input` by whole chunks. unsafe { - input_ptr = input_ptr.add(batch * CHUNK_LEN); + input_ptr = input_ptr.add(batch.strict_mul(CHUNK_LEN)); } } } @@ -392,7 +388,7 @@ fn hash_full_chunks_cvs_parallel_rayon( out: &mut [[u32; 8]], threads_total: usize, ) { - debug_assert_eq!(input.len(), out.len() * CHUNK_LEN); + debug_assert_eq!(input.len(), out.len().strict_mul(CHUNK_LEN)); if threads_total <= 1 || out.len() < 2 { hash_full_chunks_cvs_serial(kernel, key_words, flags, base_counter, input, out); @@ -400,82 +396,40 @@ fn hash_full_chunks_cvs_parallel_rayon( } let threads_total = threads_total.min(out.len()).max(1); - #[cfg(test)] - let failed = AtomicBool::new(false); let out_ptr = SendPtr(out.as_mut_ptr()); let out_len = out.len(); rayon::scope(|s| { - #[cfg(test)] - let failed = &failed; for t in 1..threads_total { let (start, end) = thread_range(t, threads_total, out_len); if start == end { continue; } - let input = &input[start * CHUNK_LEN..end * CHUNK_LEN]; + let input = &input[start.strict_mul(CHUNK_LEN)..end.strict_mul(CHUNK_LEN)]; let counter = base_counter.wrapping_add(start as u64); s.spawn(move |_| { - #[cfg(test)] - let ok = run_parallel_task(|| { + run_parallel_task(|| { // SAFETY: `out_ptr` is valid for `out_len` elements and this task's // range is disjoint from every other task. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; + let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end.strict_sub(start)) }; hash_full_chunks_cvs_serial(kernel, key_words, flags, counter, input, out); }); - #[cfg(test)] - if !ok { - failed.store(true, Ordering::Relaxed); - } - #[cfg(not(test))] - { - run_parallel_task(|| { - // SAFETY: `out_ptr` is valid for `out_len` elements and this task's - // range is disjoint from every other task. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; - hash_full_chunks_cvs_serial(kernel, key_words, flags, counter, input, out); - }); - } }); } let (start, end) = thread_range(0, threads_total, out_len); - #[cfg(test)] - let ok = run_parallel_task(|| { + run_parallel_task(|| { hash_full_chunks_cvs_serial( kernel, key_words, flags, base_counter.wrapping_add(start as u64), - &input[start * CHUNK_LEN..end * CHUNK_LEN], + &input[start.strict_mul(CHUNK_LEN)..end.strict_mul(CHUNK_LEN)], // SAFETY: disjoint partition for thread 0. - unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }, + unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end.strict_sub(start)) }, ); }); - #[cfg(test)] - if !ok { - failed.store(true, Ordering::Relaxed); - } - #[cfg(not(test))] - { - run_parallel_task(|| { - hash_full_chunks_cvs_serial( - kernel, - key_words, - flags, - base_counter.wrapping_add(start as u64), - &input[start * CHUNK_LEN..end * CHUNK_LEN], - // SAFETY: disjoint partition for thread 0. - unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }, - ); - }); - } }); - - #[cfg(test)] - if failed.load(Ordering::Relaxed) { - hash_full_chunks_cvs_serial(kernel, key_words, flags, base_counter, input, out); - } } #[cfg(feature = "parallel")] @@ -487,7 +441,7 @@ fn parent_cvs_many_from_cvs_parallel_rayon( out: &mut [[u32; 8]], threads_total: usize, ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if threads_total <= 1 || out.len() < 2 { kernels::parent_cvs_many_from_cvs_inline(kernel.id, children, key_words, flags, out); @@ -496,78 +450,37 @@ fn parent_cvs_many_from_cvs_parallel_rayon( let pairs = out.len(); let threads_total = threads_total.min(pairs).max(1); - #[cfg(test)] - let failed = AtomicBool::new(false); let out_ptr = SendPtr(out.as_mut_ptr()); rayon::scope(|s| { - #[cfg(test)] - let failed = &failed; for t in 1..threads_total { let (start, end) = thread_range(t, threads_total, pairs); if start == end { continue; } - let children = &children[2 * start..2 * end]; + let children = &children[start.strict_mul(2)..end.strict_mul(2)]; s.spawn(move |_| { - #[cfg(test)] - let ok = run_parallel_task(|| { + run_parallel_task(|| { // SAFETY: `out_ptr` is valid for `pairs` outputs and this task's // range is disjoint from every other task. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; + let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end.strict_sub(start)) }; kernels::parent_cvs_many_from_cvs_inline(kernel.id, children, key_words, flags, out); }); - #[cfg(test)] - if !ok { - failed.store(true, Ordering::Relaxed); - } - #[cfg(not(test))] - { - run_parallel_task(|| { - // SAFETY: `out_ptr` is valid for `pairs` outputs and this task's - // range is disjoint from every other task. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; - kernels::parent_cvs_many_from_cvs_inline(kernel.id, children, key_words, flags, out); - }); - } }); } let (start, end) = thread_range(0, threads_total, pairs); - #[cfg(test)] - let ok = run_parallel_task(|| { + run_parallel_task(|| { kernels::parent_cvs_many_from_cvs_inline( kernel.id, - &children[2 * start..2 * end], + &children[start.strict_mul(2)..end.strict_mul(2)], key_words, flags, // SAFETY: disjoint partition for thread 0. - unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }, + unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end.strict_sub(start)) }, ); }); - #[cfg(test)] - if !ok { - failed.store(true, Ordering::Relaxed); - } - #[cfg(not(test))] - { - run_parallel_task(|| { - kernels::parent_cvs_many_from_cvs_inline( - kernel.id, - &children[2 * start..2 * end], - key_words, - flags, - // SAFETY: disjoint partition for thread 0. - unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }, - ); - }); - } }); - - #[cfg(test)] - if failed.load(Ordering::Relaxed) { - kernels::parent_cvs_many_from_cvs_inline(kernel.id, children, key_words, flags, out); - } } #[cfg(feature = "parallel")] @@ -583,12 +496,14 @@ fn hash_power_of_two_subtree_roots_serial( ) { debug_assert!(subtree_chunks.is_power_of_two()); debug_assert_ne!(subtree_chunks, 0); - debug_assert_eq!(input.len(), out.len() * subtree_chunks * CHUNK_LEN); + debug_assert_eq!(input.len(), out.len().strict_mul(subtree_chunks).strict_mul(CHUNK_LEN)); with_subtree_scratch(subtree_chunks, flags, |scratch0, scratch1| { for (i, slot) in out.iter_mut().enumerate() { - let chunk_base = base_counter.wrapping_add((i * subtree_chunks) as u64); - let bytes = &input[i * subtree_chunks * CHUNK_LEN..(i + 1) * subtree_chunks * CHUNK_LEN]; + let chunk_offset = i.strict_mul(subtree_chunks); + let chunk_end = i.strict_add(1).strict_mul(subtree_chunks); + let chunk_base = base_counter.wrapping_add(chunk_offset as u64); + let bytes = &input[chunk_offset.strict_mul(CHUNK_LEN)..chunk_end.strict_mul(CHUNK_LEN)]; hash_full_chunks_cvs_serial(kernel, key_words, flags, chunk_base, bytes, scratch0); *slot = reduce_power_of_two_cvs_in_place(kernel, key_words, flags, scratch0, scratch1); } @@ -621,7 +536,7 @@ fn hash_power_of_two_subtree_roots_parallel_rayon(req: SubtreeRootsRequest<'_>) } = req; debug_assert!(subtree_chunks.is_power_of_two()); debug_assert_ne!(subtree_chunks, 0); - debug_assert_eq!(input.len(), out.len() * subtree_chunks * CHUNK_LEN); + debug_assert_eq!(input.len(), out.len().strict_mul(subtree_chunks).strict_mul(CHUNK_LEN)); if threads_total <= 1 || out.len() < 2 { hash_power_of_two_subtree_roots_serial(kernel, key_words, flags, base_counter, input, subtree_chunks, out); @@ -629,101 +544,58 @@ fn hash_power_of_two_subtree_roots_parallel_rayon(req: SubtreeRootsRequest<'_>) } let threads_total = threads_total.min(out.len()).max(1); - #[cfg(test)] - let failed = AtomicBool::new(false); let out_ptr = SendPtr(out.as_mut_ptr()); let out_len = out.len(); rayon::scope(|s| { - #[cfg(test)] - let failed = &failed; for t in 1..threads_total { let (start, end) = thread_range(t, threads_total, out_len); if start == end { continue; } - let input = &input[start * subtree_chunks * CHUNK_LEN..end * subtree_chunks * CHUNK_LEN]; - let counter = base_counter.wrapping_add((start * subtree_chunks) as u64); + let start_chunk = start.strict_mul(subtree_chunks); + let end_chunk = end.strict_mul(subtree_chunks); + let input = &input[start_chunk.strict_mul(CHUNK_LEN)..end_chunk.strict_mul(CHUNK_LEN)]; + let counter = base_counter.wrapping_add(start_chunk as u64); s.spawn(move |_| { - #[cfg(test)] - let ok = run_parallel_task(|| { + run_parallel_task(|| { with_subtree_scratch(subtree_chunks, flags, |scratch0, scratch1| { // SAFETY: `out_ptr` is valid for `out_len` outputs and this task's // range is disjoint from every other task. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; + let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end.strict_sub(start)) }; for (i, slot) in out.iter_mut().enumerate() { - let chunk_base = counter + (i * subtree_chunks) as u64; - let bytes = &input[i * subtree_chunks * CHUNK_LEN..(i + 1) * subtree_chunks * CHUNK_LEN]; + let chunk_offset = i.strict_mul(subtree_chunks); + let chunk_end = i.strict_add(1).strict_mul(subtree_chunks); + let chunk_base = counter.wrapping_add(chunk_offset as u64); + let bytes = &input[chunk_offset.strict_mul(CHUNK_LEN)..chunk_end.strict_mul(CHUNK_LEN)]; hash_full_chunks_cvs_serial(kernel, key_words, flags, chunk_base, bytes, scratch0); *slot = reduce_power_of_two_cvs_in_place(kernel, key_words, flags, scratch0, scratch1); } }); }); - #[cfg(test)] - if !ok { - failed.store(true, Ordering::Relaxed); - } - #[cfg(not(test))] - { - run_parallel_task(|| { - with_subtree_scratch(subtree_chunks, flags, |scratch0, scratch1| { - // SAFETY: `out_ptr` is valid for `out_len` outputs and this task's - // range is disjoint from every other task. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; - for (i, slot) in out.iter_mut().enumerate() { - let chunk_base = counter + (i * subtree_chunks) as u64; - let bytes = &input[i * subtree_chunks * CHUNK_LEN..(i + 1) * subtree_chunks * CHUNK_LEN]; - hash_full_chunks_cvs_serial(kernel, key_words, flags, chunk_base, bytes, scratch0); - *slot = reduce_power_of_two_cvs_in_place(kernel, key_words, flags, scratch0, scratch1); - } - }); - }); - } }); } let (start, end) = thread_range(0, threads_total, out_len); - #[cfg(test)] - let ok = run_parallel_task(|| { + run_parallel_task(|| { with_subtree_scratch(subtree_chunks, flags, |scratch0, scratch1| { - let counter = base_counter.wrapping_add((start * subtree_chunks) as u64); + let start_chunk = start.strict_mul(subtree_chunks); + let counter = base_counter.wrapping_add(start_chunk as u64); // SAFETY: disjoint partition for thread 0. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; + let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end.strict_sub(start)) }; for (i, slot) in out.iter_mut().enumerate() { - let chunk_base = counter + (i * subtree_chunks) as u64; - let bytes = &input[(start + i) * subtree_chunks * CHUNK_LEN..(start + i + 1) * subtree_chunks * CHUNK_LEN]; + let chunk_offset = i.strict_mul(subtree_chunks); + let global_chunk = start.strict_add(i).strict_mul(subtree_chunks); + let global_chunk_end = start.strict_add(i).strict_add(1).strict_mul(subtree_chunks); + let chunk_base = counter.wrapping_add(chunk_offset as u64); + let bytes = &input[global_chunk.strict_mul(CHUNK_LEN)..global_chunk_end.strict_mul(CHUNK_LEN)]; hash_full_chunks_cvs_serial(kernel, key_words, flags, chunk_base, bytes, scratch0); *slot = reduce_power_of_two_cvs_in_place(kernel, key_words, flags, scratch0, scratch1); } }); }); - #[cfg(test)] - if !ok { - failed.store(true, Ordering::Relaxed); - } - #[cfg(not(test))] - { - run_parallel_task(|| { - with_subtree_scratch(subtree_chunks, flags, |scratch0, scratch1| { - let counter = base_counter.wrapping_add((start * subtree_chunks) as u64); - // SAFETY: disjoint partition for thread 0. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; - for (i, slot) in out.iter_mut().enumerate() { - let chunk_base = counter + (i * subtree_chunks) as u64; - let bytes = &input[(start + i) * subtree_chunks * CHUNK_LEN..(start + i + 1) * subtree_chunks * CHUNK_LEN]; - hash_full_chunks_cvs_serial(kernel, key_words, flags, chunk_base, bytes, scratch0); - *slot = reduce_power_of_two_cvs_in_place(kernel, key_words, flags, scratch0, scratch1); - } - }); - }); - } }); - - #[cfg(test)] - if failed.load(Ordering::Relaxed) { - hash_power_of_two_subtree_roots_serial(kernel, key_words, flags, base_counter, input, subtree_chunks, out); - } } /// BLAKE3 message schedule. @@ -754,21 +626,9 @@ fn words8_from_le_bytes_32(bytes: &[u8; 32]) -> [u32; 8] { // 1-byte alignment of `[u8; 32]`. unsafe { ptr::read_unaligned(bytes.as_ptr().cast::<[u32; 8]>()) } } else { - let src = bytes.as_ptr() as *const u32; - // SAFETY: `bytes` is exactly 32 bytes; `read_unaligned` supports the - // 1-byte alignment of `[u8; 32]`. - unsafe { - [ - u32::from_le(ptr::read_unaligned(src.add(0))), - u32::from_le(ptr::read_unaligned(src.add(1))), - u32::from_le(ptr::read_unaligned(src.add(2))), - u32::from_le(ptr::read_unaligned(src.add(3))), - u32::from_le(ptr::read_unaligned(src.add(4))), - u32::from_le(ptr::read_unaligned(src.add(5))), - u32::from_le(ptr::read_unaligned(src.add(6))), - u32::from_le(ptr::read_unaligned(src.add(7))), - ] - } + let (words, remainder) = bytes.as_chunks::<4>(); + debug_assert!(remainder.is_empty()); + core::array::from_fn(|i| u32::from_le_bytes(words[i])) } } @@ -779,29 +639,9 @@ fn words16_from_le_bytes_64(bytes: &[u8; 64]) -> [u32; 16] { // 1-byte alignment of `[u8; 64]`. unsafe { ptr::read_unaligned(bytes.as_ptr().cast::<[u32; 16]>()) } } else { - let src = bytes.as_ptr() as *const u32; - // SAFETY: `bytes` is exactly 64 bytes; `read_unaligned` supports the - // 1-byte alignment of `[u8; 64]`. - unsafe { - [ - u32::from_le(ptr::read_unaligned(src.add(0))), - u32::from_le(ptr::read_unaligned(src.add(1))), - u32::from_le(ptr::read_unaligned(src.add(2))), - u32::from_le(ptr::read_unaligned(src.add(3))), - u32::from_le(ptr::read_unaligned(src.add(4))), - u32::from_le(ptr::read_unaligned(src.add(5))), - u32::from_le(ptr::read_unaligned(src.add(6))), - u32::from_le(ptr::read_unaligned(src.add(7))), - u32::from_le(ptr::read_unaligned(src.add(8))), - u32::from_le(ptr::read_unaligned(src.add(9))), - u32::from_le(ptr::read_unaligned(src.add(10))), - u32::from_le(ptr::read_unaligned(src.add(11))), - u32::from_le(ptr::read_unaligned(src.add(12))), - u32::from_le(ptr::read_unaligned(src.add(13))), - u32::from_le(ptr::read_unaligned(src.add(14))), - u32::from_le(ptr::read_unaligned(src.add(15))), - ] - } + let (words, remainder) = bytes.as_chunks::<4>(); + debug_assert!(remainder.is_empty()); + core::array::from_fn(|i| u32::from_le_bytes(words[i])) } } @@ -813,8 +653,8 @@ fn words16_to_le_bytes_64(words: &[u32; 16]) -> [u8; 64] { unsafe { ptr::copy_nonoverlapping(words.as_ptr().cast::(), out.as_mut_ptr(), 64) }; } else { for (idx, word) in words.iter().copied().enumerate() { - let offset = idx * 4; - out[offset..offset + 4].copy_from_slice(&word.to_le_bytes()); + let offset = idx.strict_mul(4); + out[offset..offset.strict_add(4)].copy_from_slice(&word.to_le_bytes()); } } out @@ -823,7 +663,7 @@ fn words16_to_le_bytes_64(words: &[u32; 16]) -> [u8; 64] { #[inline(always)] fn pow2_floor(n: usize) -> usize { debug_assert!(n != 0); - 1usize << (usize::BITS - 1 - n.leading_zeros()) + 1usize.strict_shl(usize::BITS.strict_sub(1).strict_sub(n.leading_zeros())) } #[cfg(any(target_endian = "little", feature = "parallel"))] @@ -842,9 +682,15 @@ fn reduce_power_of_two_chunk_cvs(kernel: Kernel, key_words: [u32; 8], flags: u32 let mut cur_len = cvs.len(); while cur_len > 1 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); debug_assert!(pairs <= 8); - kernels::parent_cvs_many_from_cvs_inline(kernel.id, &cur[..2 * pairs], key_words, flags, &mut next[..pairs]); + kernels::parent_cvs_many_from_cvs_inline( + kernel.id, + &cur[..2usize.strict_mul(pairs)], + key_words, + flags, + &mut next[..pairs], + ); cur[..pairs].copy_from_slice(&next[..pairs]); cur_len = pairs; } @@ -874,9 +720,15 @@ fn reduce_power_of_two_chunk_cvs_bytes(kernel: Kernel, key_words: [u32; 8], flag let mut cur_len = cvs.len(); while cur_len > 1 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); debug_assert!(pairs <= 8); - kernels::parent_cvs_many_from_bytes_inline(kernel.id, &cur[..2 * pairs], key_words, flags, &mut next[..pairs]); + kernels::parent_cvs_many_from_bytes_inline( + kernel.id, + &cur[..2usize.strict_mul(pairs)], + key_words, + flags, + &mut next[..pairs], + ); cur[..pairs].copy_from_slice(&next[..pairs]); cur_len = pairs; } @@ -918,7 +770,7 @@ fn reduce_subtree_to_pair( let mut in_next = false; // false = current data is in cvs, true = in next while cur_len > 2 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); if !in_next { kernels::parent_cvs_many_from_cvs_inline(kernel.id, &cvs[..cur_len], key_words, flags, &mut next[..pairs]); } else { @@ -956,8 +808,8 @@ fn reduce_power_of_two_chunk_cvs_any( let threads_total = threads_total.max(1); - let mut buf0 = alloc::vec![[0u32; 8]; cvs.len() / 2]; - let mut buf1 = alloc::vec![[0u32; 8]; cvs.len() / 2]; + let mut buf0 = alloc::vec![[0u32; 8]; cvs.len().strict_div(2)]; + let mut buf1 = alloc::vec![[0u32; 8]; cvs.len().strict_div(2)]; enum Cur<'a> { Input(&'a [[u32; 8]]), @@ -969,7 +821,7 @@ fn reduce_power_of_two_chunk_cvs_any( let mut cur_len = cvs.len(); let result = loop { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); debug_assert!(pairs != 0); // Parallelize only levels admitted by the explicit parent-fold threshold. @@ -998,21 +850,21 @@ fn reduce_power_of_two_chunk_cvs_any( let out0: [u32; 8] = match cur { Cur::Input(children) => { let out = &mut buf0[..pairs]; - let children = &children[..2 * pairs]; + let children = &children[..2usize.strict_mul(pairs)]; fold_level!(children, out); cur = Cur::Buf0; out[0] } Cur::Buf0 => { let out = &mut buf1[..pairs]; - let children = &buf0[..2 * pairs]; + let children = &buf0[..2usize.strict_mul(pairs)]; fold_level!(children, out); cur = Cur::Buf1; out[0] } Cur::Buf1 => { let out = &mut buf0[..pairs]; - let children = &buf1[..2 * pairs]; + let children = &buf1[..2usize.strict_mul(pairs)]; fold_level!(children, out); cur = Cur::Buf0; out[0] @@ -1053,12 +905,12 @@ fn add_chunk_cvs_batched( #[inline] fn push_stack(stack: &mut [MaybeUninit<[u32; 8]>; CV_STACK_LEN], len: &mut usize, cv: [u32; 8]) { stack[*len].write(cv); - *len += 1; + *len = len.strict_add(1); } #[inline] fn pop_stack(stack: &mut [MaybeUninit<[u32; 8]>; CV_STACK_LEN], len: &mut usize) -> [u32; 8] { - *len -= 1; + *len = len.strict_sub(1); // SAFETY: `len` tracks the number of initialized entries. unsafe { stack[*len].assume_init_read() } } @@ -1067,7 +919,7 @@ fn add_chunk_cvs_batched( let mut chunk_counter = base_counter; while offset < cvs.len() { - let remaining = cvs.len() - offset; + let remaining = cvs.len().strict_sub(offset); let mut size = pow2_floor(remaining); let aligned_max = if chunk_counter == 0 { @@ -1077,14 +929,14 @@ fn add_chunk_cvs_batched( if tz >= (usize::BITS as usize) { usize::MAX } else { - 1usize << tz + 1usize.strict_shl(u32::try_from(tz).expect("BLAKE3 tree level fits in u32")) } }; size = size.min(aligned_max).min(remaining); debug_assert!(size.is_power_of_two()); - let subtree_cv = reduce_power_of_two_chunk_cvs(kernel, key_words, flags, &cvs[offset..offset + size]); + let subtree_cv = reduce_power_of_two_chunk_cvs(kernel, key_words, flags, &cvs[offset..offset.strict_add(size)]); chunk_counter = chunk_counter.wrapping_add(size as u64); // Merge this subtree into the global stack. Because `size` is a power of two @@ -1099,7 +951,7 @@ fn add_chunk_cvs_batched( } push_stack(stack, stack_len, cv); - offset += size; + offset = offset.strict_add(size); } } @@ -1123,12 +975,12 @@ fn add_chunk_cvs_batched_bytes( #[inline] fn push_stack(stack: &mut [MaybeUninit<[u32; 8]>; CV_STACK_LEN], len: &mut usize, cv: [u32; 8]) { stack[*len].write(cv); - *len += 1; + *len = len.strict_add(1); } #[inline] fn pop_stack(stack: &mut [MaybeUninit<[u32; 8]>; CV_STACK_LEN], len: &mut usize) -> [u32; 8] { - *len -= 1; + *len = len.strict_sub(1); // SAFETY: `len` tracks the number of initialized entries. unsafe { stack[*len].assume_init_read() } } @@ -1137,7 +989,7 @@ fn add_chunk_cvs_batched_bytes( let mut chunk_counter = base_counter; while offset < cvs.len() { - let remaining = cvs.len() - offset; + let remaining = cvs.len().strict_sub(offset); let mut size = pow2_floor(remaining); let aligned_max = if chunk_counter == 0 { @@ -1147,14 +999,15 @@ fn add_chunk_cvs_batched_bytes( if tz >= (usize::BITS as usize) { usize::MAX } else { - 1usize << tz + 1usize.strict_shl(u32::try_from(tz).expect("BLAKE3 tree level fits in u32")) } }; size = size.min(aligned_max).min(remaining); debug_assert!(size.is_power_of_two()); - let subtree_cv_bytes = reduce_power_of_two_chunk_cvs_bytes(kernel, key_words, flags, &cvs[offset..offset + size]); + let subtree_cv_bytes = + reduce_power_of_two_chunk_cvs_bytes(kernel, key_words, flags, &cvs[offset..offset.strict_add(size)]); let subtree_cv = words8_from_le_bytes_32(&subtree_cv_bytes); chunk_counter = chunk_counter.wrapping_add(size as u64); @@ -1170,7 +1023,7 @@ fn add_chunk_cvs_batched_bytes( } push_stack(stack, stack_len, cv); - offset += size; + offset = offset.strict_add(size); } } @@ -1193,8 +1046,9 @@ fn compress(chaining_value: &[u32; 8], block_words: &[u32; 16], counter: u64, bl let m14 = block_words[14]; let m15 = block_words[15]; - let counter_low = counter as u32; - let counter_high = (counter >> 32) as u32; + let [c0, c1, c2, c3, c4, c5, c6, c7] = counter.to_le_bytes(); + let counter_low = u32::from_le_bytes([c0, c1, c2, c3]); + let counter_high = u32::from_le_bytes([c4, c5, c6, c7]); let mut v0 = chaining_value[0]; let mut v1 = chaining_value[1]; let mut v2 = chaining_value[2]; @@ -1290,8 +1144,8 @@ fn words8_to_le_bytes(words: &[u32; 8]) -> [u8; OUT_LEN] { unsafe { ptr::copy_nonoverlapping(words.as_ptr().cast::(), out.as_mut_ptr(), OUT_LEN) }; } else { for (i, word) in words.iter().copied().enumerate() { - let offset = i * 4; - out[offset..offset + 4].copy_from_slice(&word.to_le_bytes()); + let offset = i.strict_mul(4); + out[offset..offset.strict_add(4)].copy_from_slice(&word.to_le_bytes()); } } out @@ -1355,15 +1209,13 @@ impl OutputState { #[inline] fn into_root_emit_state(self) -> RootEmitState { - debug_assert!(self.block_len <= u8::MAX as u32); - debug_assert!(self.flags <= u8::MAX as u32); RootEmitState { kernel_id: self.kernel_id, input_chaining_value: self.input_chaining_value, block_bytes: words16_to_le_bytes_64(&self.block_words), counter: self.counter, - block_len: self.block_len as u8, - flags: self.flags as u8, + block_len: u8::try_from(self.block_len).expect("BLAKE3 block length fits in u8"), + flags: u8::try_from(self.flags).expect("BLAKE3 flags fit in u8"), } } } @@ -1406,8 +1258,8 @@ impl RootEmitState { input_chaining_value: key_words, block_bytes, counter: 0, - block_len: BLOCK_LEN as u8, - flags: (PARENT | flags) as u8, + block_len: u8::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u8"), + flags: u8::try_from(PARENT | flags).expect("BLAKE3 flags fit in u8"), }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words_no_fence(&mut left_child_cv); @@ -1439,11 +1291,11 @@ impl RootEmitState { "digest prefix fast path only applies to the first output block" ); debug_assert!(offset <= OUT_LEN); - debug_assert!(offset + out.len() <= OUT_LEN); + debug_assert!(offset.strict_add(out.len()) <= OUT_LEN); let mut block = [0u8; OUTPUT_BLOCK_LEN]; self.emit_one_block(&mut block); - out.copy_from_slice(&block[offset..offset + out.len()]); + out.copy_from_slice(&block[offset..offset.strict_add(out.len())]); if u32::from(self.flags) & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize(&mut block); } @@ -1531,7 +1383,9 @@ impl ChunkState { #[inline] fn len(&self) -> usize { - BLOCK_LEN * self.blocks_compressed as usize + self.block_len as usize + BLOCK_LEN + .strict_mul(usize::from(self.blocks_compressed)) + .strict_add(usize::from(self.block_len)) } #[inline] @@ -1551,11 +1405,11 @@ impl ChunkState { self.chunk_counter, self.flags, &mut self.blocks_compressed, - &input[..CHUNK_LEN - BLOCK_LEN], + &input[..CHUNK_LEN.strict_sub(BLOCK_LEN)], ); debug_assert_eq!(self.blocks_compressed, 15); - self.block.copy_from_slice(&input[CHUNK_LEN - BLOCK_LEN..]); - self.block_len = BLOCK_LEN as u8; + self.block.copy_from_slice(&input[CHUNK_LEN.strict_sub(BLOCK_LEN)..]); + self.block_len = u8::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u8"); } #[inline] @@ -1567,7 +1421,7 @@ impl ChunkState { let (cv, last_block) = absorb_exact_one_chunk_state(self.kernel_id, input, key, self.chunk_counter, self.flags); self.chaining_value = cv; self.block = last_block; - self.block_len = BLOCK_LEN as u8; + self.block_len = u8::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u8"); self.blocks_compressed = 15; } @@ -1579,7 +1433,7 @@ impl ChunkState { if self.block_len == 0 { if input.len() <= BLOCK_LEN { self.block[..input.len()].copy_from_slice(input); - self.block_len = input.len() as u8; + self.block_len = u8::try_from(input.len()).expect("BLAKE3 buffered block length fits in u8"); return; } @@ -1600,7 +1454,7 @@ impl ChunkState { // Calls assembly directly, bypassing the multi-block wrappers and their // #[target_feature] boundaries. This matches the official blake3 crate's // call depth: one match + one extern "C" assembly call. - if self.block_len as usize == BLOCK_LEN && self.blocks_compressed < 15 && input.len() == BLOCK_LEN { + if usize::from(self.block_len) == BLOCK_LEN && self.blocks_compressed < 15 && input.len() == BLOCK_LEN { let start = if self.blocks_compressed == 0 { CHUNK_START } else { 0 }; // SAFETY: kernel_id was validated at construction to match available CPU features. unsafe { @@ -1612,7 +1466,7 @@ impl ChunkState { self.flags | start, ); } - self.blocks_compressed = self.blocks_compressed.wrapping_add(1); + self.blocks_compressed = self.blocks_compressed.strict_add(1); self.block.copy_from_slice(input); // block_len stays BLOCK_LEN — no zero-fill needed return; @@ -1626,10 +1480,12 @@ impl ChunkState { // (or, if it's already full, compress it) before touching the caller // slice. This keeps the hot "many full blocks" path branch-light. if self.block_len != 0 { - let want = BLOCK_LEN - self.block_len as usize; + let want = BLOCK_LEN.strict_sub(usize::from(self.block_len)); let take = min(want, input.len()); - self.block[self.block_len as usize..][..take].copy_from_slice(&input[..take]); - self.block_len = self.block_len.strict_add(take as u8); + self.block[usize::from(self.block_len)..][..take].copy_from_slice(&input[..take]); + self.block_len = self + .block_len + .strict_add(u8::try_from(take).expect("BLAKE3 buffered byte count fits in u8")); input = &input[take..]; // If the caller ended mid-block, we're done. Note that this also covers @@ -1640,7 +1496,7 @@ impl ChunkState { return; } - if self.block_len as usize == BLOCK_LEN { + if usize::from(self.block_len) == BLOCK_LEN { debug_assert!( self.blocks_compressed < 15, "last chunk block stays buffered until output()" @@ -1656,7 +1512,7 @@ impl ChunkState { self.flags | start, ); } - self.blocks_compressed = self.blocks_compressed.wrapping_add(1); + self.blocks_compressed = self.blocks_compressed.strict_add(1); self.block_len = 0; self.block = [0u8; BLOCK_LEN]; } @@ -1671,13 +1527,13 @@ impl ChunkState { if self.blocks_compressed == 15 { debug_assert!(input.len() <= BLOCK_LEN); self.block[..input.len()].copy_from_slice(input); - self.block_len = input.len() as u8; + self.block_len = u8::try_from(input.len()).expect("BLAKE3 buffered block length fits in u8"); return; } - let full_blocks = input.len() / BLOCK_LEN; + let full_blocks = input.len().strict_div(BLOCK_LEN); if full_blocks != 0 { - let max_blocks = 15usize - self.blocks_compressed as usize; + let max_blocks = 15usize.strict_sub(usize::from(self.blocks_compressed)); let mut blocks_to_compress = full_blocks.min(max_blocks); // If we'd consume the entire input as full blocks, leave one block @@ -1701,7 +1557,7 @@ impl ChunkState { self.flags | start, ); } - self.blocks_compressed = self.blocks_compressed.wrapping_add(1); + self.blocks_compressed = self.blocks_compressed.strict_add(1); } input = &input[bytes..]; continue; @@ -1711,7 +1567,7 @@ impl ChunkState { // Remainder: buffer <= 64 bytes and return. let take = min(BLOCK_LEN, input.len()); self.block[..take].copy_from_slice(&input[..take]); - self.block_len = take as u8; + self.block_len = u8::try_from(take).expect("BLAKE3 buffered block length fits in u8"); return; } } @@ -1723,7 +1579,7 @@ impl ChunkState { self.chaining_value, &self.block, self.chunk_counter, - self.block_len as u32, + u32::from(self.block_len), self.flags | self.start_flag() | CHUNK_END, ) } @@ -1740,7 +1596,7 @@ impl ChunkState { block_bytes: self.block, counter: self.chunk_counter, block_len: self.block_len, - flags: (self.flags | self.start_flag() | CHUNK_END) as u8, + flags: u8::try_from(self.flags | self.start_flag() | CHUNK_END).expect("BLAKE3 flags fit in u8"), } } @@ -1751,7 +1607,7 @@ impl ChunkState { input_chaining_value: self.chaining_value, block_words: words16_from_le_bytes_64(&self.block), counter: self.chunk_counter, - block_len: self.block_len as u32, + block_len: u32::from(self.block_len), flags: self.flags | self.start_flag() | CHUNK_END, } } @@ -1769,7 +1625,7 @@ fn absorb_exact_one_chunk_state( #[cfg(target_arch = "x86_64")] { - let (prefix_blocks, remainder) = input[..CHUNK_LEN - BLOCK_LEN].as_chunks::(); + let (prefix_blocks, remainder) = input[..CHUNK_LEN.strict_sub(BLOCK_LEN)].as_chunks::(); debug_assert!(remainder.is_empty()); match kernel_id { @@ -1780,11 +1636,17 @@ fn absorb_exact_one_chunk_state( // SAFETY: dispatch validates SSE4.1 for both kernels, and `block` is a // readable 64-byte buffer. unsafe { - x86_64::compress_in_place_sse41_bytes(&mut cv, block.as_ptr(), counter, BLOCK_LEN as u32, block_flags) + x86_64::compress_in_place_sse41_bytes( + &mut cv, + block.as_ptr(), + counter, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + block_flags, + ) }; } let mut last_block = [0u8; BLOCK_LEN]; - last_block.copy_from_slice(&input[CHUNK_LEN - BLOCK_LEN..]); + last_block.copy_from_slice(&input[CHUNK_LEN.strict_sub(BLOCK_LEN)..]); return (cv, last_block); } kernels::Blake3KernelId::X86Avx512 => { @@ -1796,7 +1658,13 @@ fn absorb_exact_one_chunk_state( // SAFETY: dispatch validates AVX-512 availability, and `block` is a // readable 64-byte buffer. unsafe { - x86_64::asm::compress_in_place_avx512_mut(&mut cv, block.as_ptr(), counter, BLOCK_LEN as u32, block_flags) + x86_64::asm::compress_in_place_avx512_mut( + &mut cv, + block.as_ptr(), + counter, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + block_flags, + ) }; } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] @@ -1804,12 +1672,18 @@ fn absorb_exact_one_chunk_state( // SAFETY: dispatch validates AVX-512 availability, and `block` is a // readable 64-byte buffer. unsafe { - x86_64::compress_in_place_avx512_bytes(&mut cv, block.as_ptr(), counter, BLOCK_LEN as u32, block_flags) + x86_64::compress_in_place_avx512_bytes( + &mut cv, + block.as_ptr(), + counter, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + block_flags, + ) }; } } let mut last_block = [0u8; BLOCK_LEN]; - last_block.copy_from_slice(&input[CHUNK_LEN - BLOCK_LEN..]); + last_block.copy_from_slice(&input[CHUNK_LEN.strict_sub(BLOCK_LEN)..]); return (cv, last_block); } _ => {} @@ -1847,11 +1721,11 @@ fn absorb_exact_one_chunk_state( counter, flags, &mut blocks_compressed, - &input[..CHUNK_LEN - BLOCK_LEN], + &input[..CHUNK_LEN.strict_sub(BLOCK_LEN)], ); debug_assert_eq!(blocks_compressed, 15); let mut last_block = [0u8; BLOCK_LEN]; - last_block.copy_from_slice(&input[CHUNK_LEN - BLOCK_LEN..]); + last_block.copy_from_slice(&input[CHUNK_LEN.strict_sub(BLOCK_LEN)..]); (cv, last_block) } @@ -1893,7 +1767,7 @@ fn parent_output( input_chaining_value: key_words, block_words, counter: 0, - block_len: BLOCK_LEN as u32, + block_len: u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), flags: PARENT | flags, }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { @@ -1945,7 +1819,7 @@ fn single_chunk_output( input_chaining_value: cv_words, block_words, counter: chunk_counter, - block_len: BLOCK_LEN as u32, + block_len: u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), flags: flags | CHUNK_END, }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { @@ -1965,14 +1839,14 @@ fn single_chunk_output( let (full_blocks, last_len) = if input.is_empty() { (0usize, 0usize) } else if input.len().is_multiple_of(BLOCK_LEN) { - (blocks - 1, BLOCK_LEN) + (blocks.strict_sub(1), BLOCK_LEN) } else { - (blocks - 1, input.len() % BLOCK_LEN) + (blocks.strict_sub(1), input.len().strict_rem(BLOCK_LEN)) }; let mut chaining_value = key_words; let mut blocks_compressed: u8 = 0; - let full_bytes = full_blocks * BLOCK_LEN; + let full_bytes = full_blocks.strict_mul(BLOCK_LEN); (kernel.chunk_compress_blocks)( &mut chaining_value, chunk_counter, @@ -1984,7 +1858,7 @@ fn single_chunk_output( let mut block_words = if cfg!(target_endian = "little") { let mut out = [0u32; 16]; if !input.is_empty() { - let offset = full_blocks * BLOCK_LEN; + let offset = full_blocks.strict_mul(BLOCK_LEN); // SAFETY: `out` is 64 bytes, and `last_len <= 64`. unsafe { ptr::copy_nonoverlapping(input.as_ptr().add(offset), out.as_mut_ptr().cast::(), last_len); @@ -1994,8 +1868,8 @@ fn single_chunk_output( } else { let mut last_block = [0u8; BLOCK_LEN]; if !input.is_empty() { - let offset = full_blocks * BLOCK_LEN; - last_block[..last_len].copy_from_slice(&input[offset..offset + last_len]); + let offset = full_blocks.strict_mul(BLOCK_LEN); + last_block[..last_len].copy_from_slice(&input[offset..offset.strict_add(last_len)]); } let block_words = words16_from_le_bytes_64(&last_block); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { @@ -2010,7 +1884,7 @@ fn single_chunk_output( input_chaining_value: chaining_value, block_words, counter: chunk_counter, - block_len: last_len as u32, + block_len: u32::try_from(last_len).expect("BLAKE3 block length fits in u32"), flags: flags | start | CHUNK_END, }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { @@ -2039,8 +1913,8 @@ fn root_output_oneshot( return output; } - let full_chunks = input.len() / CHUNK_LEN; - let remainder = input.len() % CHUNK_LEN; + let full_chunks = input.len().strict_div(CHUNK_LEN); + let remainder = input.len().strict_rem(CHUNK_LEN); const MAX_SIMD_DEGREE: usize = 16; const FAST_TREE_MAX_CHUNKS: usize = 128; @@ -2053,7 +1927,11 @@ fn root_output_oneshot( // // This is intentionally conservative to avoid overhead on latency-critical // small inputs (including keyed/derive). - let commit_full_chunks = if remainder == 0 { full_chunks - 1 } else { full_chunks }; + let commit_full_chunks = if remainder == 0 { + full_chunks.strict_sub(1) + } else { + full_chunks + }; if let Some(threads) = control::parallel_policy_threads_with_admission(mode, input.len(), full_chunks, commit_full_chunks) { @@ -2072,7 +1950,7 @@ fn root_output_oneshot( #[cfg(target_endian = "little")] { let mut cur = [[0u32; 8]; MAX_SIMD_DEGREE]; - let mut next = [[0u32; 8]; MAX_SIMD_DEGREE / 2]; + let mut next = [[0u32; 8]; MAX_SIMD_DEGREE.strict_div(2)]; // SAFETY: input has exactly `full_chunks * CHUNK_LEN` bytes and `cur` has // `full_chunks` CV slots (`full_chunks * OUT_LEN` bytes). @@ -2090,7 +1968,7 @@ fn root_output_oneshot( let mut cur_len = full_chunks; let mut cur_is_cur = true; while cur_len > 2 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); if cur_is_cur { kernels::parent_cvs_many_from_cvs_inline(kernel.id, &cur[..cur_len], key_words, flags, &mut next[..pairs]); } else { @@ -2116,7 +1994,7 @@ fn root_output_oneshot( #[cfg(not(target_endian = "little"))] { let mut cur = [[0u8; OUT_LEN]; MAX_SIMD_DEGREE]; - let mut next = [[0u8; OUT_LEN]; MAX_SIMD_DEGREE / 2]; + let mut next = [[0u8; OUT_LEN]; MAX_SIMD_DEGREE.strict_div(2)]; // SAFETY: input has exactly `full_chunks * CHUNK_LEN` bytes and `cur` has // `full_chunks` CV slots (`full_chunks * OUT_LEN` bytes). @@ -2134,7 +2012,7 @@ fn root_output_oneshot( let mut cur_len = full_chunks; let mut cur_is_cur = true; while cur_len > 2 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); if cur_is_cur { kernels::parent_cvs_many_from_bytes_inline( kernel.id, @@ -2177,7 +2055,7 @@ fn root_output_oneshot( #[cfg(target_endian = "little")] { let mut cur = [[0u32; 8]; FAST_TREE_MAX_CHUNKS]; - let mut next = [[0u32; 8]; FAST_TREE_MAX_CHUNKS / 2]; + let mut next = [[0u32; 8]; FAST_TREE_MAX_CHUNKS.strict_div(2)]; // SAFETY: input has exactly `full_chunks * CHUNK_LEN` bytes and `cur` has // `full_chunks` CV slots (`full_chunks * OUT_LEN` bytes). @@ -2195,7 +2073,7 @@ fn root_output_oneshot( let mut cur_len = full_chunks; let mut cur_is_cur = true; while cur_len > 2 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); if cur_is_cur { kernels::parent_cvs_many_from_cvs_inline(kernel.id, &cur[..cur_len], key_words, flags, &mut next[..pairs]); } else { @@ -2221,7 +2099,7 @@ fn root_output_oneshot( #[cfg(not(target_endian = "little"))] { let mut cur = [[0u8; OUT_LEN]; FAST_TREE_MAX_CHUNKS]; - let mut next = [[0u8; OUT_LEN]; FAST_TREE_MAX_CHUNKS / 2]; + let mut next = [[0u8; OUT_LEN]; FAST_TREE_MAX_CHUNKS.strict_div(2)]; // SAFETY: input has exactly `full_chunks * CHUNK_LEN` bytes and `cur` has // `full_chunks` CV slots (`full_chunks * OUT_LEN` bytes). @@ -2239,7 +2117,7 @@ fn root_output_oneshot( let mut cur_len = full_chunks; let mut cur_is_cur = true; while cur_len > 2 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); if cur_is_cur { kernels::parent_cvs_many_from_bytes_inline( kernel.id, @@ -2291,7 +2169,7 @@ fn root_output_oneshot( let mut chunk_counter = 0u64; let mut offset = 0usize; while chunk_counter < full_chunks as u64 { - let remaining = (full_chunks as u64).strict_sub(chunk_counter) as usize; + let remaining = full_chunks.strict_sub(offset.strict_div(CHUNK_LEN)); let batch = core::cmp::min(remaining, MAX_SIMD_DEGREE); debug_assert!(batch != 0); @@ -2310,8 +2188,8 @@ fn root_output_oneshot( let mut commit = batch; if remainder == 0 && chunk_counter.strict_add(batch as u64) == full_chunks as u64 { - last_full_chunk_cv = Some(cvs[batch - 1]); - commit -= 1; + last_full_chunk_cv = Some(cvs[batch.strict_sub(1)]); + commit = commit.strict_sub(1); } if commit != 0 { @@ -2331,7 +2209,7 @@ fn root_output_oneshot( } let right_cv = if remainder != 0 { - let chunk_bytes = &input[full_chunks * CHUNK_LEN..]; + let chunk_bytes = &input[full_chunks.strict_mul(CHUNK_LEN)..]; single_chunk_output(kernel, key_words, full_chunks as u64, flags, chunk_bytes).chaining_value() } else { last_full_chunk_cv @@ -2357,7 +2235,7 @@ fn root_output_oneshot( let mut chunk_counter = 0u64; let mut offset = 0usize; while chunk_counter < full_chunks as u64 { - let remaining = (full_chunks as u64).strict_sub(chunk_counter) as usize; + let remaining = full_chunks.strict_sub(offset.strict_div(CHUNK_LEN)); let batch = core::cmp::min(remaining, MAX_SIMD_DEGREE); debug_assert!(batch != 0); @@ -2375,8 +2253,8 @@ fn root_output_oneshot( let mut commit = batch; if remainder == 0 && chunk_counter.strict_add(batch as u64) == full_chunks as u64 { - last_full_chunk_cv = Some(cvs[batch - 1]); - commit -= 1; + last_full_chunk_cv = Some(cvs[batch.strict_sub(1)]); + commit = commit.strict_sub(1); } if commit != 0 { @@ -2396,7 +2274,7 @@ fn root_output_oneshot( } let right_cv = if remainder != 0 { - let chunk_bytes = &input[full_chunks * CHUNK_LEN..]; + let chunk_bytes = &input[full_chunks.strict_mul(CHUNK_LEN)..]; single_chunk_output(kernel, key_words, full_chunks as u64, flags, chunk_bytes).chaining_value() } else if let Some(cv) = last_full_chunk_cv.as_ref() { words8_from_le_bytes_32(cv) @@ -2416,7 +2294,7 @@ fn root_output_oneshot( let mut parent_nodes_remaining = cv_stack_len; debug_assert!(parent_nodes_remaining > 0); - parent_nodes_remaining -= 1; + parent_nodes_remaining = parent_nodes_remaining.strict_sub(1); // SAFETY: `cv_stack_len` tracks the number of initialized entries. let mut left = unsafe { cv_stack[parent_nodes_remaining].assume_init_read() }; let mut output = parent_output(kernel.id, left, right_cv, key_words, flags); @@ -2425,7 +2303,7 @@ fn root_output_oneshot( ct::zeroize_words_no_fence(&mut right_cv); } while parent_nodes_remaining > 0 { - parent_nodes_remaining -= 1; + parent_nodes_remaining = parent_nodes_remaining.strict_sub(1); // SAFETY: `cv_stack_len` tracks the number of initialized entries. let mut left = unsafe { cv_stack[parent_nodes_remaining].assume_init_read() }; let mut right = output.chaining_value(); @@ -2458,8 +2336,8 @@ fn recompute_last_full_chunk_cv( debug_assert!(full_chunks != 0); debug_assert_eq!(input.len() % CHUNK_LEN, 0); - let last_chunk_index = full_chunks.saturating_sub(1); - let offset = input.len().saturating_sub(CHUNK_LEN); + let last_chunk_index = full_chunks.strict_sub(1); + let offset = input.len().strict_sub(CHUNK_LEN); let cv = single_chunk_output(kernel, key_words, last_chunk_index as u64, flags, &input[offset..]).chaining_value(); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words(&mut key_words); @@ -2502,7 +2380,7 @@ fn hash_full_chunks_cvs_scoped( threads_total: usize, ) { let out_len = out.len(); - debug_assert_eq!(input.len(), out_len * CHUNK_LEN); + debug_assert_eq!(input.len(), out_len.strict_mul(CHUNK_LEN)); #[cfg(feature = "parallel")] { @@ -2628,6 +2506,7 @@ fn digest_public_oneshot(mut key_words: [u32; 8], flags: u32, input: &[u8]) -> [ } #[cfg(feature = "diag")] +/// Computes the constant-time evidence digest with the portable kernel. #[must_use] pub fn diag_blake3_keyed_digest_portable(key: &[u8; KEY_LEN]) -> Blake3KeyedHash { let mut key_words = words8_from_le_bytes_32(key); @@ -2682,34 +2561,47 @@ pub fn diag_zeroize_blake3_xof_consume(mut reader: Blake3XofReader) -> u8 { } #[cfg(feature = "diag")] +/// BLAKE3 kernels exposed for diagnostic comparison. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Blake3DiagKernel { + /// Portable Rust kernel. Portable, #[cfg(target_arch = "x86_64")] + /// x86-64 SSE4.1 kernel. X86Sse41, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX2 kernel. X86Avx2, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX2 kernel with the owned hash-many implementation. X86Avx2OwnedHashMany, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX2 kernel with paired chunk-tail processing. X86Avx2PairChunkTail, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX2 kernel with paired parent-tail processing. X86Avx2PairParentTail, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX-512 kernel. X86Avx512, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX-512 kernel forced through exact-block assembly. X86Avx512ExactBlockAsm, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX-512 kernel with the owned hash-many implementation. X86Avx512OwnedHashMany, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX-512 kernel with the owned compression implementation. X86Avx512OwnedCompress, #[cfg(target_arch = "aarch64")] + /// AArch64 NEON kernel. Aarch64Neon, } #[cfg(feature = "diag")] impl Blake3DiagKernel { #[inline] + /// Returns the stable diagnostic label for this kernel. #[must_use] pub const fn label(self) -> &'static str { match self { @@ -2738,6 +2630,7 @@ impl Blake3DiagKernel { } #[inline] + /// Returns whether this kernel supports streaming diagnostics. #[must_use] pub const fn supports_streaming(self) -> bool { #[cfg(target_arch = "x86_64")] @@ -2825,6 +2718,7 @@ impl Blake3DiagKernel { #[cfg(feature = "diag")] #[inline] +/// Returns whether the current CPU supports the diagnostic kernel. #[must_use] pub fn diag_blake3_kernel_available(kernel: Blake3DiagKernel) -> bool { #[cfg(target_arch = "x86_64")] @@ -2870,6 +2764,7 @@ fn diag_blake3_kernel(kernel: Blake3DiagKernel) -> Option { } #[cfg(feature = "diag")] +/// Computes a digest with a selected diagnostic kernel. #[must_use] pub fn diag_blake3_digest_with_kernel(kernel: Blake3DiagKernel, data: &[u8]) -> Option<[u8; OUT_LEN]> { let kernel = diag_blake3_kernel(kernel)?; @@ -2877,6 +2772,7 @@ pub fn diag_blake3_digest_with_kernel(kernel: Blake3DiagKernel, data: &[u8]) -> } #[cfg(feature = "diag")] +/// Computes a keyed digest with a selected diagnostic kernel. #[must_use] pub fn diag_blake3_keyed_digest_with_kernel( kernel: Blake3DiagKernel, @@ -2891,6 +2787,7 @@ pub fn diag_blake3_keyed_digest_with_kernel( } #[cfg(feature = "diag")] +/// Fills XOF output with a selected diagnostic kernel. pub fn diag_blake3_xof_with_kernel(kernel: Blake3DiagKernel, data: &[u8], out: &mut [u8]) -> Option<()> { let kernel = diag_blake3_kernel(kernel)?; let mut reader = if data.len() <= CHUNK_LEN { @@ -2909,6 +2806,7 @@ pub fn diag_blake3_xof_with_kernel(kernel: Blake3DiagKernel, data: &[u8], out: & } #[cfg(feature = "diag")] +/// Computes a streaming digest with a selected diagnostic kernel and update size. #[must_use] pub fn diag_blake3_streaming_digest_with_kernel( kernel: Blake3DiagKernel, @@ -2931,14 +2829,15 @@ pub fn diag_blake3_streaming_digest_with_kernel( } #[cfg(feature = "diag")] +/// Writes full-chunk chaining values with a selected diagnostic kernel. pub fn diag_blake3_chunk_cvs_with_kernel(kernel: Blake3DiagKernel, data: &[u8], out: &mut [u8]) -> Option<()> { #[cfg(target_arch = "x86_64")] let requested_kernel = kernel; if data.is_empty() || !data.len().is_multiple_of(CHUNK_LEN) { return None; } - let chunks = data.len() / CHUNK_LEN; - if out.len() != chunks * OUT_LEN { + let chunks = data.len().strict_div(CHUNK_LEN); + if out.len() != chunks.strict_mul(OUT_LEN) { return None; } @@ -2959,6 +2858,7 @@ pub fn diag_blake3_chunk_cvs_with_kernel(kernel: Blake3DiagKernel, data: &[u8], } #[cfg(feature = "diag")] +/// Writes parent chaining values with a selected diagnostic kernel. pub fn diag_blake3_parent_cvs_with_kernel(kernel: Blake3DiagKernel, children: &[u8], out: &mut [u8]) -> Option<()> { #[cfg(target_arch = "x86_64")] let requested_kernel = kernel; @@ -2967,7 +2867,7 @@ pub fn diag_blake3_parent_cvs_with_kernel(kernel: Blake3DiagKernel, children: &[ return None; } - let parent_count = children.len() / 2; + let parent_count = children.len().strict_div(2); let (out, out_remainder) = out.as_chunks_mut::(); if !out_remainder.is_empty() || out.len() != parent_count { return None; @@ -3314,13 +3214,17 @@ impl Blake3 { }; let keep_last_full_chunk = input.len().is_multiple_of(CHUNK_LEN) && batch == full_chunks; - let commit = if keep_last_full_chunk { batch - 1 } else { batch }; + let commit = if keep_last_full_chunk { + batch.strict_sub(1) + } else { + batch + }; if commit != 0 { // SAFETY: `out_buf` stores `batch` contiguous CV outputs, and // `commit <= batch <= MAX_SIMD_DEGREE`. let cvs_bytes: &[[u8; OUT_LEN]] = unsafe { slice::from_raw_parts(out_buf.as_ptr().cast::<[u8; OUT_LEN]>(), commit) }; - let mut stack_len = self.cv_stack_len as usize; + let mut stack_len = usize::from(self.cv_stack_len); add_chunk_cvs_batched_bytes( kernels::kernel(self.bulk_kernel_id), &mut self.cv_stack, @@ -3330,10 +3234,10 @@ impl Blake3 { self.key_words, self.chunk_state.flags, ); - self.cv_stack_len = stack_len as u8; + self.cv_stack_len = u8::try_from(stack_len).expect("BLAKE3 CV stack length is at most 64"); } - let new_counter = base_counter.strict_add(batch as u64); + let new_counter = base_counter.strict_add(u64::try_from(batch).expect("BLAKE3 SIMD batch size fits in u64")); self.chunk_state = ChunkState::new( self.key_words, new_counter, @@ -3344,7 +3248,7 @@ impl Blake3 { let offset = batch.strict_sub(1).strict_mul(OUT_LEN); // SAFETY: `out_buf` is `OUT_LEN * MAX_SIMD_DEGREE`, and `offset` // is `(batch - 1) * OUT_LEN` with `batch <= MAX_SIMD_DEGREE`. - let mut cv = unsafe { words8_from_le_bytes_32(&*(out_buf.as_ptr().add(offset) as *const [u8; OUT_LEN])) }; + let mut cv = unsafe { words8_from_le_bytes_32(&*out_buf.as_ptr().add(offset).cast::<[u8; OUT_LEN]>()) }; self.pending_chunk_cv = Some(cv); self.pending_cv_chunks = 1; if self.chunk_state.flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { @@ -3412,7 +3316,7 @@ impl Blake3 { continue; } - let want = CHUNK_LEN - self.chunk_state.len(); + let want = CHUNK_LEN.strict_sub(self.chunk_state.len()); let take = min(want, input.len()); self.chunk_state.update(&input[..take]); input = &input[take..]; @@ -3445,8 +3349,8 @@ impl Blake3 { #[inline] fn push_stack(&mut self, mut cv: [u32; 8]) { - self.cv_stack[self.cv_stack_len as usize].write(cv); - self.cv_stack_len = self.cv_stack_len.wrapping_add(1); + self.cv_stack[usize::from(self.cv_stack_len)].write(cv); + self.cv_stack_len = self.cv_stack_len.strict_add(1); if self.chunk_state.flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words(&mut cv); } @@ -3454,13 +3358,13 @@ impl Blake3 { #[inline] fn pop_stack(&mut self) -> [u32; 8] { - self.cv_stack_len = self.cv_stack_len.wrapping_sub(1); + self.cv_stack_len = self.cv_stack_len.strict_sub(1); // SAFETY: `cv_stack_len` tracks the number of initialized entries. - let cv = unsafe { self.cv_stack[self.cv_stack_len as usize].assume_init_read() }; + let cv = unsafe { self.cv_stack[usize::from(self.cv_stack_len)].assume_init_read() }; if self.chunk_state.flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { // SAFETY: the slot held the `Copy` CV read above and remains initialized // until the next push overwrites it. - ct::zeroize_words(unsafe { self.cv_stack[self.cv_stack_len as usize].assume_init_mut() }); + ct::zeroize_words(unsafe { self.cv_stack[usize::from(self.cv_stack_len)].assume_init_mut() }); } cv } @@ -3593,7 +3497,7 @@ impl Blake3 { self.advance_full_chunk(); } - if self.chunk_state.len() + input.len() <= CHUNK_LEN { + if self.chunk_state.len().strict_add(input.len()) <= CHUNK_LEN { self.chunk_state.update(input); return; } @@ -3630,7 +3534,7 @@ impl Blake3 { }; while parent_nodes_remaining > 0 { - parent_nodes_remaining -= 1; + parent_nodes_remaining = parent_nodes_remaining.strict_sub(1); // SAFETY: `cv_stack_len` tracks the number of initialized entries. let mut left = unsafe { *self.cv_stack[parent_nodes_remaining].assume_init_ref() }; let mut right = output.chaining_value(); @@ -3702,7 +3606,7 @@ impl Blake3 { }; while parent_nodes_remaining > 0 { - parent_nodes_remaining -= 1; + parent_nodes_remaining = parent_nodes_remaining.strict_sub(1); // SAFETY: `cv_stack_len` tracks the number of initialized entries. let mut left = unsafe { *self.cv_stack[parent_nodes_remaining].assume_init_ref() }; if parent_nodes_remaining == 0 { @@ -3920,8 +3824,8 @@ fn xof_oneshot_single_chunk(kernel: Kernel, mut key_words: [u32; 8], flags: u32, input_chaining_value: key_words, block_bytes, counter: 0, - block_len: input.len() as u8, - flags: (flags | CHUNK_START | CHUNK_END) as u8, + block_len: u8::try_from(input.len()).expect("BLAKE3 block length fits in u8"), + flags: u8::try_from(flags | CHUNK_START | CHUNK_END).expect("BLAKE3 flags fit in u8"), }); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_no_fence(&mut block_bytes); @@ -3946,24 +3850,29 @@ fn xof_oneshot_single_chunk(kernel: Kernel, mut key_words: [u32; 8], flags: u32, } // Single-chunk input (65–1024B): compress full blocks, store last block. - let rem = input.len() % BLOCK_LEN; + let rem = input.len().strict_rem(BLOCK_LEN); let (full_blocks, last_len) = if rem == 0 { - (input.len() / BLOCK_LEN - 1, BLOCK_LEN) + (input.len().strict_div(BLOCK_LEN).strict_sub(1), BLOCK_LEN) } else { - (input.len() / BLOCK_LEN, rem) + (input.len().strict_div(BLOCK_LEN), rem) }; let mut cv = key_words; + let (input_blocks, _) = input.as_chunks::(); // Compress full non-final blocks using the kernel's compress function // pointer directly. Single-chunk XOF is latency-sensitive, and the bulk // wrapper setup costs more than it saves in the 256B–1KiB range. - for i in 0..full_blocks { - let offset = i * BLOCK_LEN; + for (i, block) in input_blocks[..full_blocks].iter().enumerate() { let block_flags = flags | if i == 0 { CHUNK_START } else { 0 }; - // SAFETY: `offset + BLOCK_LEN <= input.len()` by construction. - let mut block_words = unsafe { words16_from_le_bytes_64(&*input.as_ptr().add(offset).cast::<[u8; BLOCK_LEN]>()) }; - cv = first_8_words((kernel.compress)(&cv, &block_words, 0, BLOCK_LEN as u32, block_flags)); + let mut block_words = words16_from_le_bytes_64(block); + cv = first_8_words((kernel.compress)( + &cv, + &block_words, + 0, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + block_flags, + )); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words(&mut block_words); } @@ -3972,16 +3881,16 @@ fn xof_oneshot_single_chunk(kernel: Kernel, mut key_words: [u32; 8], flags: u32, // Stage the final block bytes for the root-output reader. let start = if full_blocks == 0 { CHUNK_START } else { 0 }; let mut block_bytes = [0u8; BLOCK_LEN]; - let offset = full_blocks * BLOCK_LEN; - block_bytes[..last_len].copy_from_slice(&input[offset..offset + last_len]); + let offset = full_blocks.strict_mul(BLOCK_LEN); + block_bytes[..last_len].copy_from_slice(&input[offset..offset.strict_add(last_len)]); let reader = Blake3XofReader::new(RootEmitState { kernel_id: kernel.id, input_chaining_value: cv, block_bytes, counter: 0, - block_len: last_len as u8, - flags: (flags | start | CHUNK_END) as u8, + block_len: u8::try_from(last_len).expect("BLAKE3 block length fits in u8"), + flags: u8::try_from(flags | start | CHUNK_END).expect("BLAKE3 flags fit in u8"), }); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words_no_fence(&mut cv); @@ -4036,17 +3945,21 @@ impl Blake3XofReader { #[inline] fn fill_root_hash_prefix(&mut self, out: &mut &mut [u8]) { - let offset = self.position_within_block as usize; - let take = min(out.len(), OUT_LEN - offset); + let offset = usize::from(self.position_within_block); + let take = min(out.len(), OUT_LEN.strict_sub(offset)); self.root.emit_root_hash_prefix(offset, &mut out[..take]); - self.position_within_block += take as u8; + self.position_within_block = self + .position_within_block + .strict_add(u8::try_from(take).expect("BLAKE3 output position fits in u8")); *out = &mut core::mem::take(out)[take..]; } #[inline] fn fill_one_block(&mut self, out: &mut &mut [u8]) { - if self.root.counter == 0 && self.position_within_block < OUT_LEN as u8 { - let digest_bytes_remaining = OUT_LEN - self.position_within_block as usize; + if self.root.counter == 0 + && self.position_within_block < u8::try_from(OUT_LEN).expect("BLAKE3 digest length fits in u8") + { + let digest_bytes_remaining = OUT_LEN.strict_sub(usize::from(self.position_within_block)); if out.len() <= digest_bytes_remaining { self.fill_root_hash_prefix(out); return; @@ -4055,11 +3968,13 @@ impl Blake3XofReader { let mut block = [0u8; OUTPUT_BLOCK_LEN]; self.root.emit_one_block(&mut block); - let output_bytes = &block[self.position_within_block as usize..]; + let output_bytes = &block[usize::from(self.position_within_block)..]; let take = min(out.len(), output_bytes.len()); out[..take].copy_from_slice(&output_bytes[..take]); - self.position_within_block += take as u8; - if self.position_within_block == OUTPUT_BLOCK_LEN as u8 { + self.position_within_block = self + .position_within_block + .strict_add(u8::try_from(take).expect("BLAKE3 output position fits in u8")); + if self.position_within_block == u8::try_from(OUTPUT_BLOCK_LEN).expect("BLAKE3 output block length fits in u8") { self.root.counter = self.root.counter.wrapping_add(1); self.position_within_block = 0; } @@ -4080,11 +3995,14 @@ impl Blake3XofReader { self.fill_one_block(out); } - let full = out.len() / OUTPUT_BLOCK_LEN * OUTPUT_BLOCK_LEN; + let blocks = out.len().strict_div(OUTPUT_BLOCK_LEN); + let full = blocks.strict_mul(OUTPUT_BLOCK_LEN); if full != 0 { - let blocks = (full / OUTPUT_BLOCK_LEN) as u64; self.root.emit_blocks_into(&mut out[..full]); - self.root.counter = self.root.counter.wrapping_add(blocks); + self.root.counter = self + .root + .counter + .wrapping_add(u64::try_from(blocks).expect("BLAKE3 output block count fits in u64")); *out = &mut core::mem::take(out)[full..]; } @@ -4169,7 +4087,15 @@ fn compress_chunk_tail_to_root_words( kernels::Blake3KernelId::X86Sse41 | kernels::Blake3KernelId::X86Avx2 | kernels::Blake3KernelId::X86Avx512 => { // SAFETY: dispatch validates required CPU features before selecting // each x86 kernel; `block` is a readable 64-byte buffer. - let output = unsafe { (kernel.x86_compress_cv_bytes)(&cv, block.as_ptr(), 0, block_len as u32, final_flags) }; + let output = unsafe { + (kernel.x86_compress_cv_bytes)( + &cv, + block.as_ptr(), + 0, + u32::try_from(block_len).expect("BLAKE3 block length fits in u32"), + final_flags, + ) + }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words(&mut cv); } @@ -4183,7 +4109,15 @@ fn compress_chunk_tail_to_root_words( { if kernel.id == kernels::Blake3KernelId::Aarch64Neon { // SAFETY: NEON availability validated by dispatch - let output = unsafe { aarch64::compress_cv_neon_bytes(&cv, block.as_ptr(), 0, block_len as u32, final_flags) }; + let output = unsafe { + aarch64::compress_cv_neon_bytes( + &cv, + block.as_ptr(), + 0, + u32::try_from(block_len).expect("BLAKE3 block length fits in u32"), + final_flags, + ) + }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words(&mut cv); } @@ -4193,7 +4127,13 @@ fn compress_chunk_tail_to_root_words( // Portable fallback let mut block_words = words16_from_le_bytes_64(block); - let output = first_8_words((kernel.compress)(&cv, &block_words, 0, block_len as u32, final_flags)); + let output = first_8_words((kernel.compress)( + &cv, + &block_words, + 0, + u32::try_from(block_len).expect("BLAKE3 block length fits in u32"), + final_flags, + )); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words_no_fence(&mut cv); ct::zeroize_words_no_fence(&mut block_words); @@ -4240,26 +4180,31 @@ fn digest_one_chunk_root_hash_words_generic( let (full_blocks, last_len) = if input.is_empty() { (0usize, 0usize) } else { - let rem = input.len() % BLOCK_LEN; + let rem = input.len().strict_rem(BLOCK_LEN); if rem == 0 { - (input.len() / BLOCK_LEN - 1, BLOCK_LEN) + (input.len().strict_div(BLOCK_LEN).strict_sub(1), BLOCK_LEN) } else { - (input.len() / BLOCK_LEN, rem) + (input.len().strict_div(BLOCK_LEN), rem) } }; let mut cv = key_words; + let (input_blocks, _) = input.as_chunks::(); // Process full non-final blocks. Use the compress function pointer directly // instead of chunk_compress_blocks_inline to avoid the kernel-ID match // dispatch on every call — the Kernel struct already carries the resolved // function pointer. - for i in 0..full_blocks { - let offset = i * BLOCK_LEN; + for (i, block) in input_blocks[..full_blocks].iter().enumerate() { let block_flags = flags | if i == 0 { CHUNK_START } else { 0 }; - // SAFETY: `offset + BLOCK_LEN <= input.len()` by construction. - let mut block_words = unsafe { words16_from_le_bytes_64(&*input.as_ptr().add(offset).cast::<[u8; BLOCK_LEN]>()) }; - cv = first_8_words((kernel.compress)(&cv, &block_words, 0, BLOCK_LEN as u32, block_flags)); + let mut block_words = words16_from_le_bytes_64(block); + cv = first_8_words((kernel.compress)( + &cv, + &block_words, + 0, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + block_flags, + )); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words(&mut block_words); } @@ -4269,10 +4214,14 @@ fn digest_one_chunk_root_hash_words_generic( let final_flags = flags | start | CHUNK_END | ROOT; if last_len == BLOCK_LEN && !input.is_empty() { - let offset = full_blocks * BLOCK_LEN; - // SAFETY: `offset + BLOCK_LEN <= input.len()` by construction. - let mut block_words = unsafe { words16_from_le_bytes_64(&*input.as_ptr().add(offset).cast::<[u8; BLOCK_LEN]>()) }; - let output = first_8_words((kernel.compress)(&cv, &block_words, 0, BLOCK_LEN as u32, final_flags)); + let mut block_words = words16_from_le_bytes_64(&input_blocks[full_blocks]); + let output = first_8_words((kernel.compress)( + &cv, + &block_words, + 0, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + final_flags, + )); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words_no_fence(&mut block_words); ct::zeroize_words_no_fence(&mut cv); @@ -4284,13 +4233,18 @@ fn digest_one_chunk_root_hash_words_generic( let mut final_block = [0u8; BLOCK_LEN]; if last_len != 0 { - let offset = full_blocks * BLOCK_LEN; - // SAFETY: `last_len < BLOCK_LEN` here, and source range is in-bounds. - unsafe { ptr::copy_nonoverlapping(input.as_ptr().add(offset), final_block.as_mut_ptr(), last_len) }; + let offset = full_blocks.strict_mul(BLOCK_LEN); + final_block[..last_len].copy_from_slice(&input[offset..offset.strict_add(last_len)]); } let mut final_words = words16_from_le_bytes_64(&final_block); - let output = first_8_words((kernel.compress)(&cv, &final_words, 0, last_len as u32, final_flags)); + let output = first_8_words((kernel.compress)( + &cv, + &final_words, + 0, + u32::try_from(last_len).expect("BLAKE3 block length fits in u32"), + final_flags, + )); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_no_fence(&mut final_block); ct::zeroize_words_no_fence(&mut final_words); @@ -4310,7 +4264,10 @@ fn use_avx2_hash_many_one_chunk_fast_path() -> bool { dispatch::avx2_hash_many_one_chunk_fast_path() } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(target_os = "linux", target_os = "macos", target_os = "windows") +))] #[inline] #[must_use] fn use_avx512_four_block_avx2_fast_path() -> bool { @@ -4343,6 +4300,12 @@ fn use_x86_hash_many_exact_block_one_chunk_fast_path(kernel: Kernel, input_len: #[cfg(target_arch = "x86_64")] #[inline] +/// Compresses an exact-block one-chunk prefix with AVX2. +/// +/// # Safety +/// +/// The current CPU must support AVX2, SSE4.1, and SSSE3. `input` must be +/// nonempty, no longer than one chunk, and contain only complete blocks. unsafe fn avx2_owned_exact_block_chain( mut key_words: [u32; 8], flags: u32, @@ -4360,7 +4323,7 @@ unsafe fn avx2_owned_exact_block_chain( if block_idx == 0 { block_flags |= CHUNK_START; } - if block_idx + 1 == blocks { + if block_idx.strict_add(1) == blocks { block_flags |= final_extra_flags; } @@ -4372,9 +4335,9 @@ unsafe fn avx2_owned_exact_block_chain( cv = unsafe { x86_64::compress_cv_avx2_bytes( &cv, - input.as_ptr().add(block_idx * BLOCK_LEN), + input.as_ptr().add(block_idx.strict_mul(BLOCK_LEN)), 0, - BLOCK_LEN as u32, + BLOCK_LEN_U32, block_flags, ) }; @@ -4390,6 +4353,13 @@ unsafe fn avx2_owned_exact_block_chain( #[cfg(all(feature = "diag", target_arch = "x86_64"))] #[inline] +/// Hashes one exact-block input through every AVX-512 lane. +/// +/// # Safety +/// +/// The current CPU must support the AVX-512 and AVX2 features required by +/// `hash16_owned`. `input` must be nonempty, no longer than one chunk, and +/// contain only complete blocks. unsafe fn avx512_owned_exact_block_hash_many( mut key_words: [u32; 8], flags: u32, @@ -4410,17 +4380,17 @@ unsafe fn avx512_owned_exact_block_hash_many( // 2. Every duplicated lane points at `input`, which is readable for `blocks * BLOCK_LEN` bytes. // 3. `out` is writable for the full 16-lane output contract; only lane 0 is returned. unsafe { - x86_64::avx512::hash16_owned( - &inputs, + x86_64::avx512::hash16_owned(x86_64::HashManyRequest { + inputs: &inputs, blocks, - &key_words, - 0, - false, + key: &key_words, + counter: 0, + increment_counter: false, flags, - CHUNK_START, - final_extra_flags, - out.as_mut_ptr(), - ); + flags_start: CHUNK_START, + flags_end: final_extra_flags, + out: out.as_mut_ptr(), + }); } let mut lane0 = [0u8; OUT_LEN]; @@ -4437,6 +4407,12 @@ unsafe fn avx512_owned_exact_block_hash_many( #[cfg(target_arch = "x86_64")] #[inline] +/// Builds an XOF reader for an exact-block x86 one-chunk input. +/// +/// # Safety +/// +/// The caller must have validated the target features required by `kernel`. +/// `input` must contain two through sixteen complete blocks. unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( kernel: Kernel, mut key_words: [u32; 8], @@ -4454,15 +4430,20 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( return None; } - let prefix_blocks = input.len() / BLOCK_LEN - 1; + let prefix_blocks = (input.len() / BLOCK_LEN).strict_sub(1); debug_assert!(prefix_blocks != 0); - debug_assert!(flags <= u8::MAX as u32); - debug_assert!((flags | CHUNK_START) <= u8::MAX as u32); + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] + let flags_u8 = u8::try_from(flags).expect("BLAKE3 flags fit in u8"); + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] + let flags_start_u8 = u8::try_from(flags | CHUNK_START).expect("BLAKE3 start flags fit in u8"); + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] let mut cv_bytes = [0u8; OUT_LEN]; + #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] + let mut cv_bytes: [u8; OUT_LEN]; match kernel.id { kernels::Blake3KernelId::X86Avx2 => { - let prefix_len = prefix_blocks * BLOCK_LEN; + let prefix_len = prefix_blocks.strict_mul(BLOCK_LEN); // SAFETY: AVX2 dispatch selected this kernel. `prefix_len` is non-zero, // exact-block aligned, and within `input`. let mut cv = unsafe { avx2_owned_exact_block_chain(key_words, flags, &input[..prefix_len], 0) }; @@ -4474,7 +4455,7 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( kernels::Blake3KernelId::X86Avx512 => { #[cfg(feature = "diag")] if kernel.owned_x86_hash_many { - let prefix_len = prefix_blocks * BLOCK_LEN; + let prefix_len = prefix_blocks.strict_mul(BLOCK_LEN); // SAFETY: Diagnostic availability checked the owned AVX-512 hash-many feature set, and the prefix // is exact-block aligned. let mut cv = unsafe { avx512_owned_exact_block_hash_many(key_words, flags, &input[..prefix_len], 0) }; @@ -4483,23 +4464,24 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( ct::zeroize_words(&mut cv); } } else { + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] let input_ptrs = [input.as_ptr()]; #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] // SAFETY: this branch only runs for the AVX-512 kernel on supported OSes, // with one in-bounds exact-block input lane and a valid 32-byte output buffer. unsafe { - x86_64::asm::hash_many_avx512( - input_ptrs.as_ptr(), - 1, - prefix_blocks, - key_words.as_ptr(), - 0, - false, - flags as u8, - (flags | CHUNK_START) as u8, - flags as u8, - cv_bytes.as_mut_ptr(), - ); + x86_64::asm::hash_many_avx512(x86_64::asm::HashManyRequest { + inputs: input_ptrs.as_ptr(), + num_inputs: 1, + blocks: prefix_blocks, + key: key_words.as_ptr(), + counter: 0, + increment_counter: false, + flags: flags_u8, + flags_start: flags_start_u8, + flags_end: flags_u8, + out: cv_bytes.as_mut_ptr(), + }); } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] { @@ -4511,23 +4493,24 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( } #[cfg(not(feature = "diag"))] { + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] let input_ptrs = [input.as_ptr()]; #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] // SAFETY: this branch only runs for the AVX-512 kernel on supported OSes, // with one in-bounds exact-block input lane and a valid 32-byte output buffer. unsafe { - x86_64::asm::hash_many_avx512( - input_ptrs.as_ptr(), - 1, - prefix_blocks, - key_words.as_ptr(), - 0, - false, - flags as u8, - (flags | CHUNK_START) as u8, - flags as u8, - cv_bytes.as_mut_ptr(), - ); + x86_64::asm::hash_many_avx512(x86_64::asm::HashManyRequest { + inputs: input_ptrs.as_ptr(), + num_inputs: 1, + blocks: prefix_blocks, + key: key_words.as_ptr(), + counter: 0, + increment_counter: false, + flags: flags_u8, + flags_start: flags_start_u8, + flags_end: flags_u8, + out: cv_bytes.as_mut_ptr(), + }); } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] { @@ -4547,7 +4530,7 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( } let mut block_bytes = [0u8; BLOCK_LEN]; - let last_offset = prefix_blocks * BLOCK_LEN; + let last_offset = prefix_blocks.strict_mul(BLOCK_LEN); // SAFETY: `last_offset + BLOCK_LEN <= input.len()` by construction. unsafe { ptr::copy_nonoverlapping(input.as_ptr().add(last_offset), block_bytes.as_mut_ptr(), BLOCK_LEN); @@ -4558,8 +4541,8 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( input_chaining_value: words8_from_le_bytes_32(&cv_bytes), block_bytes, counter: 0, - block_len: BLOCK_LEN as u8, - flags: (flags | CHUNK_END) as u8, + block_len: BLOCK_LEN_U8, + flags: u8::try_from(flags | CHUNK_END).expect("BLAKE3 end flags fit in u8"), }); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_no_fence(&mut cv_bytes); @@ -4572,6 +4555,12 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( #[cfg(target_arch = "x86_64")] #[inline] +/// Hashes a one-chunk input with the selected x86 kernel. +/// +/// # Safety +/// +/// The caller must have validated the target features required by `kernel`, +/// and `input` must not exceed one chunk. unsafe fn digest_one_chunk_root_hash_words_x86( kernel: Kernel, mut key_words: [u32; 8], @@ -4587,9 +4576,6 @@ unsafe fn digest_one_chunk_root_hash_words_x86( { let blocks = input.len() / BLOCK_LEN; debug_assert!((1..=CHUNK_LEN / BLOCK_LEN).contains(&blocks)); - debug_assert!(flags <= u8::MAX as u32); - debug_assert!((flags | CHUNK_START) <= u8::MAX as u32); - debug_assert!((flags | CHUNK_END | ROOT) <= u8::MAX as u32); // SAFETY: AVX2 dispatch selected this kernel, and this branch is restricted // to non-empty exact-block one-chunk input. let output = unsafe { avx2_owned_exact_block_chain(key_words, flags, input, CHUNK_END | ROOT) }; @@ -4607,9 +4593,9 @@ unsafe fn digest_one_chunk_root_hash_words_x86( debug_assert!((1..=CHUNK_LEN / BLOCK_LEN).contains(&blocks)); let flags_start = flags | CHUNK_START; let flags_end = flags | CHUNK_END | ROOT; - debug_assert!(flags <= u8::MAX as u32); - debug_assert!(flags_start <= u8::MAX as u32); - debug_assert!(flags_end <= u8::MAX as u32); + let flags_u8 = u8::try_from(flags).expect("BLAKE3 flags fit in u8"); + let flags_start_u8 = u8::try_from(flags_start).expect("BLAKE3 start flags fit in u8"); + let flags_end_u8 = u8::try_from(flags_end).expect("BLAKE3 root flags fit in u8"); #[cfg(feature = "diag")] if kernel.owned_x86_hash_many { // SAFETY: Diagnostic availability checked the owned AVX-512 hash-many feature set, and `input` is @@ -4641,18 +4627,18 @@ unsafe fn digest_one_chunk_root_hash_words_x86( // SAFETY: AVX-512 dispatch selected this kernel; input is one contiguous // full-chunk-or-less buffer; output points to one OUT_LEN digest lane. unsafe { - x86_64::asm::hash_many_avx512( - input_ptrs.as_ptr(), - 1, + x86_64::asm::hash_many_avx512(x86_64::asm::HashManyRequest { + inputs: input_ptrs.as_ptr(), + num_inputs: 1, blocks, - key_words.as_ptr(), - 0, - false, - flags as u8, - flags_start as u8, - flags_end as u8, - out.as_mut_ptr(), - ); + key: key_words.as_ptr(), + counter: 0, + increment_counter: false, + flags: flags_u8, + flags_start: flags_start_u8, + flags_end: flags_end_u8, + out: out.as_mut_ptr(), + }); } let output = words8_from_le_bytes_32(&out); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { @@ -4670,7 +4656,7 @@ unsafe fn digest_one_chunk_root_hash_words_x86( } else { let rem = input.len() % BLOCK_LEN; if rem == 0 { - (input.len() / BLOCK_LEN - 1, BLOCK_LEN) + ((input.len() / BLOCK_LEN).strict_sub(1), BLOCK_LEN) } else { (input.len() / BLOCK_LEN, rem) } @@ -4679,7 +4665,7 @@ unsafe fn digest_one_chunk_root_hash_words_x86( // Hash all full blocks except the final block, updating the CV. This keeps // ROOT out of the dependency chain until the last compress. let mut cv = key_words; - let full_bytes = full_blocks * BLOCK_LEN; + let full_bytes = full_blocks.strict_mul(BLOCK_LEN); if full_blocks != 0 { let first_block_ptr = input.as_ptr(); let first_flags = flags | CHUNK_START; @@ -4688,13 +4674,13 @@ unsafe fn digest_one_chunk_root_hash_words_x86( unsafe { match kernel.id { kernels::Blake3KernelId::X86Sse41 => { - x86_64::compress_in_place_sse41_bytes(&mut cv, first_block_ptr, 0, BLOCK_LEN as u32, first_flags); + x86_64::compress_in_place_sse41_bytes(&mut cv, first_block_ptr, 0, BLOCK_LEN_U32, first_flags); } kernels::Blake3KernelId::X86Avx2 => { - x86_64::compress_in_place_avx2_bytes(&mut cv, first_block_ptr, 0, BLOCK_LEN as u32, first_flags); + x86_64::compress_in_place_avx2_bytes(&mut cv, first_block_ptr, 0, BLOCK_LEN_U32, first_flags); } kernels::Blake3KernelId::X86Avx512 => { - cv = (kernel.x86_compress_cv_bytes)(&cv, first_block_ptr, 0, BLOCK_LEN as u32, first_flags); + cv = (kernel.x86_compress_cv_bytes)(&cv, first_block_ptr, 0, BLOCK_LEN_U32, first_flags); } _ => { let output = digest_one_chunk_root_hash_words_generic(kernel, key_words, flags, input); @@ -4719,18 +4705,18 @@ unsafe fn digest_one_chunk_root_hash_words_x86( if last_len == BLOCK_LEN && !input.is_empty() { // SAFETY: `full_blocks * BLOCK_LEN + BLOCK_LEN <= input.len()`. - let block_ptr = unsafe { input.as_ptr().add(full_blocks * BLOCK_LEN) }; + let block_ptr = unsafe { input.as_ptr().add(full_blocks.strict_mul(BLOCK_LEN)) }; // SAFETY: x86 dispatch selected this function only for x86 SIMD kernels. unsafe { match kernel.id { kernels::Blake3KernelId::X86Sse41 => { - x86_64::compress_in_place_sse41_bytes(&mut cv, block_ptr, 0, BLOCK_LEN as u32, final_flags); + x86_64::compress_in_place_sse41_bytes(&mut cv, block_ptr, 0, BLOCK_LEN_U32, final_flags); } kernels::Blake3KernelId::X86Avx2 => { - x86_64::compress_in_place_avx2_bytes(&mut cv, block_ptr, 0, BLOCK_LEN as u32, final_flags); + x86_64::compress_in_place_avx2_bytes(&mut cv, block_ptr, 0, BLOCK_LEN_U32, final_flags); } kernels::Blake3KernelId::X86Avx512 => { - cv = (kernel.x86_compress_cv_bytes)(&cv, block_ptr, 0, BLOCK_LEN as u32, final_flags); + cv = (kernel.x86_compress_cv_bytes)(&cv, block_ptr, 0, BLOCK_LEN_U32, final_flags); } _ => { let output = digest_one_chunk_root_hash_words_generic(kernel, key_words, flags, input); @@ -4755,7 +4741,7 @@ unsafe fn digest_one_chunk_root_hash_words_x86( // Partial final block (including empty): pad to 64 bytes. let mut padded = [0u8; BLOCK_LEN]; if last_len != 0 { - let offset = full_blocks * BLOCK_LEN; + let offset = full_blocks.strict_mul(BLOCK_LEN); // SAFETY: `padded` is 64 bytes, and `last_len < 64` here. unsafe { ptr::copy_nonoverlapping(input.as_ptr().add(offset), padded.as_mut_ptr(), last_len) }; } @@ -4765,13 +4751,31 @@ unsafe fn digest_one_chunk_root_hash_words_x86( unsafe { match kernel.id { kernels::Blake3KernelId::X86Sse41 => { - x86_64::compress_in_place_sse41_bytes(&mut cv, block_ptr, 0, last_len as u32, final_flags); + x86_64::compress_in_place_sse41_bytes( + &mut cv, + block_ptr, + 0, + u32::try_from(last_len).expect("BLAKE3 block length fits in u32"), + final_flags, + ); } kernels::Blake3KernelId::X86Avx2 => { - x86_64::compress_in_place_avx2_bytes(&mut cv, block_ptr, 0, last_len as u32, final_flags); + x86_64::compress_in_place_avx2_bytes( + &mut cv, + block_ptr, + 0, + u32::try_from(last_len).expect("BLAKE3 block length fits in u32"), + final_flags, + ); } kernels::Blake3KernelId::X86Avx512 => { - cv = (kernel.x86_compress_cv_bytes)(&cv, block_ptr, 0, last_len as u32, final_flags); + cv = (kernel.x86_compress_cv_bytes)( + &cv, + block_ptr, + 0, + u32::try_from(last_len).expect("BLAKE3 block length fits in u32"), + final_flags, + ); } _ => { let output = digest_one_chunk_root_hash_words_generic(kernel, key_words, flags, input); @@ -4799,6 +4803,12 @@ unsafe fn digest_one_chunk_root_hash_words_x86( #[cfg(target_arch = "aarch64")] #[inline] +/// Hashes a one-chunk input with the selected AArch64 NEON kernel. +/// +/// # Safety +/// +/// The caller must ensure that the runtime CPU supports the features required +/// by `kernel` and that `kernel` is the AArch64 NEON implementation. unsafe fn digest_one_chunk_root_hash_words_aarch64( kernel: Kernel, mut key_words: [u32; 8], @@ -4813,22 +4823,30 @@ unsafe fn digest_one_chunk_root_hash_words_aarch64( let (full_blocks, last_len) = if input.is_empty() { (0usize, 0usize) } else { - let rem = input.len() % BLOCK_LEN; + let rem = input.len().strict_rem(BLOCK_LEN); if rem == 0 { - (input.len() / BLOCK_LEN - 1, BLOCK_LEN) + (input.len().strict_div(BLOCK_LEN).strict_sub(1), BLOCK_LEN) } else { - (input.len() / BLOCK_LEN, rem) + (input.len().strict_div(BLOCK_LEN), rem) } }; // Hash all full blocks except the final block, updating the CV. This keeps // ROOT out of the dependency chain until the last compress. let mut cv = key_words; - let full_bytes = full_blocks * BLOCK_LEN; + let full_bytes = full_blocks.strict_mul(BLOCK_LEN); if full_blocks != 0 { // SAFETY: `input` covers at least one full 64-byte block here, and this // helper is only selected when NEON support has already been validated. - cv = unsafe { aarch64::compress_cv_neon_bytes(&cv, input.as_ptr(), 0, BLOCK_LEN as u32, flags | CHUNK_START) }; + cv = unsafe { + aarch64::compress_cv_neon_bytes( + &cv, + input.as_ptr(), + 0, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + flags | CHUNK_START, + ) + }; if full_blocks > 1 { let mut blocks_compressed: u8 = 1; kernels::chunk_compress_blocks_inline( @@ -4849,10 +4867,10 @@ unsafe fn digest_one_chunk_root_hash_words_aarch64( let mut padded = [0u8; BLOCK_LEN]; let block_ptr = if last_len == BLOCK_LEN && !input.is_empty() { // SAFETY: `full_blocks * BLOCK_LEN + BLOCK_LEN <= input.len()`. - unsafe { input.as_ptr().add(full_blocks * BLOCK_LEN) } + unsafe { input.as_ptr().add(full_blocks.strict_mul(BLOCK_LEN)) } } else { if last_len != 0 { - let offset = full_blocks * BLOCK_LEN; + let offset = full_blocks.strict_mul(BLOCK_LEN); // SAFETY: `padded` is 64 bytes, and `last_len < 64` here. unsafe { ptr::copy_nonoverlapping(input.as_ptr().add(offset), padded.as_mut_ptr(), last_len) }; } @@ -4861,7 +4879,15 @@ unsafe fn digest_one_chunk_root_hash_words_aarch64( // SAFETY: `block_ptr` points to 64 bytes (either into `input` or `padded`), // and dispatch only selects NEON when the required CPU features are present. - let output = unsafe { aarch64::compress_cv_neon_bytes(&cv, block_ptr, 0, last_len as u32, final_flags) }; + let output = unsafe { + aarch64::compress_cv_neon_bytes( + &cv, + block_ptr, + 0, + u32::try_from(last_len).expect("BLAKE3 block length fits in u32"), + final_flags, + ) + }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_no_fence(&mut padded); ct::zeroize_words_no_fence(&mut cv); @@ -4873,7 +4899,9 @@ unsafe fn digest_one_chunk_root_hash_words_aarch64( #[cfg(test)] mod tests { - use super::{Blake3, Blake3KeyedHash, CHUNK_LEN, OUT_LEN}; + #[cfg(feature = "diag")] + use super::CHUNK_LEN; + use super::{Blake3, Blake3KeyedHash, OUT_LEN}; use crate::traits::{Digest, VerificationError, Xof}; #[test] @@ -4940,14 +4968,16 @@ mod tests { for len in [0usize, 1, 64, 256, 1024, 4096, 8192, 16384] { let input = input_pattern(len); - let digest = diag_blake3_digest_with_kernel(kernel, &input).unwrap(); + let digest = + diag_blake3_digest_with_kernel(kernel, &input).expect("available diagnostic kernel returns a digest"); assert_eq!(digest, *blake3::hash(&input).as_bytes(), "diag digest mismatch"); - let keyed = diag_blake3_keyed_digest_with_kernel(kernel, KEY, &input).unwrap(); + let keyed = diag_blake3_keyed_digest_with_kernel(kernel, KEY, &input) + .expect("available diagnostic kernel returns a digest"); assert_eq!(keyed.to_bytes(), *blake3::keyed_hash(KEY, &input).as_bytes()); let mut xof = [0u8; 64]; - diag_blake3_xof_with_kernel(kernel, &input, &mut xof).unwrap(); + diag_blake3_xof_with_kernel(kernel, &input, &mut xof).expect("available diagnostic kernel returns XOF output"); let mut expected_xof = [0u8; 64]; let mut official = blake3::Hasher::new(); official.update(&input); @@ -4955,7 +4985,8 @@ mod tests { assert_eq!(xof, expected_xof, "diag xof mismatch"); if kernel.supports_streaming() { - let streaming = diag_blake3_streaming_digest_with_kernel(kernel, &input, 64).unwrap(); + let streaming = diag_blake3_streaming_digest_with_kernel(kernel, &input, 64) + .expect("streaming-capable diagnostic kernel returns a digest"); assert_eq!(streaming, *blake3::hash(&input).as_bytes(), "diag streaming mismatch"); } } @@ -4997,16 +5028,18 @@ mod tests { ]; for &chunks in RAW_CV_COUNTS { - let input = input_pattern(chunks * CHUNK_LEN); - let mut expected = alloc::vec![0u8; chunks * OUT_LEN]; - diag_blake3_chunk_cvs_with_kernel(Blake3DiagKernel::Portable, &input, &mut expected).unwrap(); + let input = input_pattern(chunks.strict_mul(CHUNK_LEN)); + let mut expected = alloc::vec![0u8; chunks.strict_mul(OUT_LEN)]; + diag_blake3_chunk_cvs_with_kernel(Blake3DiagKernel::Portable, &input, &mut expected) + .expect("portable diagnostic kernel returns chunk chaining values"); for &kernel in KERNELS { if !diag_blake3_kernel_available(kernel) { continue; } - let mut actual = alloc::vec![0u8; chunks * OUT_LEN]; - diag_blake3_chunk_cvs_with_kernel(kernel, &input, &mut actual).unwrap(); + let mut actual = alloc::vec![0u8; chunks.strict_mul(OUT_LEN)]; + diag_blake3_chunk_cvs_with_kernel(kernel, &input, &mut actual) + .expect("available diagnostic kernel returns chunk chaining values"); assert_eq!( actual, expected, "diag chunk CV mismatch kernel={kernel:?} chunks={chunks}" @@ -5015,16 +5048,18 @@ mod tests { } for &parents in RAW_CV_COUNTS { - let children = input_pattern(parents * 2 * OUT_LEN); - let mut expected = alloc::vec![0u8; parents * OUT_LEN]; - diag_blake3_parent_cvs_with_kernel(Blake3DiagKernel::Portable, &children, &mut expected).unwrap(); + let children = input_pattern(parents.strict_mul(2).strict_mul(OUT_LEN)); + let mut expected = alloc::vec![0u8; parents.strict_mul(OUT_LEN)]; + diag_blake3_parent_cvs_with_kernel(Blake3DiagKernel::Portable, &children, &mut expected) + .expect("portable diagnostic kernel returns parent chaining values"); for &kernel in KERNELS { if !diag_blake3_kernel_available(kernel) { continue; } - let mut actual = alloc::vec![0u8; parents * OUT_LEN]; - diag_blake3_parent_cvs_with_kernel(kernel, &children, &mut actual).unwrap(); + let mut actual = alloc::vec![0u8; parents.strict_mul(OUT_LEN)]; + diag_blake3_parent_cvs_with_kernel(kernel, &children, &mut actual) + .expect("available diagnostic kernel returns parent chaining values"); assert_eq!( actual, expected, "diag parent CV mismatch kernel={kernel:?} parents={parents}" @@ -5037,18 +5072,24 @@ mod tests { const CONTEXT: &str = "BLAKE3 2019-12-27 16:29:52 test vectors context"; fn hex_to_bytes(hex: &str, out: &mut [u8]) { - assert_eq!(hex.len(), out.len() * 2); - for (i, chunk) in hex.as_bytes().chunks_exact(2).enumerate() { - let hi = (chunk[0] as char).to_digit(16).unwrap(); - let lo = (chunk[1] as char).to_digit(16).unwrap(); - out[i] = ((hi << 4) | lo) as u8; + assert_eq!(hex.len(), out.len().strict_mul(2)); + let (hex_bytes, remainder) = hex.as_bytes().as_chunks::<2>(); + assert!(remainder.is_empty()); + for (i, chunk) in hex_bytes.iter().enumerate() { + let hi = char::from(chunk[0]) + .to_digit(16) + .expect("test vector contains hexadecimal digits"); + let lo = char::from(chunk[1]) + .to_digit(16) + .expect("test vector contains hexadecimal digits"); + out[i] = u8::try_from(hi.strict_shl(4) | lo).expect("two hexadecimal digits fit in u8"); } } fn input_pattern(len: usize) -> alloc::vec::Vec { let mut v = alloc::vec::Vec::with_capacity(len); for i in 0..len { - v.push((i % 251) as u8); + v.push(u8::try_from(i.strict_rem(251)).expect("input pattern byte fits in u8")); } v } diff --git a/src/hashes/crypto/blake3/parallel.rs b/src/hashes/crypto/blake3/parallel.rs index 1324a24d..d83fe213 100644 --- a/src/hashes/crypto/blake3/parallel.rs +++ b/src/hashes/crypto/blake3/parallel.rs @@ -47,7 +47,7 @@ impl Drop for ParallelBatchScratch { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_blake3_parallel_scratch(input: [u32; 8]) -> u32 { +pub(super) fn diag_zeroize_blake3_parallel_scratch(input: [u32; 8]) -> u32 { let mut scratch = ParallelBatchScratch::default(); scratch.zeroize = true; scratch.roots.push(input); @@ -76,7 +76,7 @@ impl Blake3 { #[inline] fn push_stack(stack: &mut [MaybeUninit<[u32; 8]>; CV_STACK_LEN], len: &mut usize, mut cv: [u32; 8], zeroize: bool) { stack[*len].write(cv); - *len += 1; + *len = len.strict_add(1); if zeroize { ct::zeroize_words(&mut cv); } @@ -84,7 +84,7 @@ impl Blake3 { #[inline] fn pop_stack(stack: &mut [MaybeUninit<[u32; 8]>; CV_STACK_LEN], len: &mut usize, zeroize: bool) -> [u32; 8] { - *len -= 1; + *len = len.strict_sub(1); // SAFETY: `len` tracks the number of initialized entries. let cv = unsafe { stack[*len].assume_init_read() }; if zeroize { @@ -125,22 +125,28 @@ impl Blake3 { size = size.min(aligned_max).min(remaining_commit); debug_assert!(size.is_power_of_two()); - let bytes_base = offset_chunks * CHUNK_LEN; - let subtree_bytes = size * CHUNK_LEN; - let subtree_input = &batch_input[bytes_base..bytes_base + subtree_bytes]; + let bytes_base = offset_chunks.strict_mul(CHUNK_LEN); + let subtree_bytes = size.strict_mul(CHUNK_LEN); + let subtree_end = bytes_base.strict_add(subtree_bytes); + let subtree_input = &batch_input[bytes_base..subtree_end]; let bulk_kernel = kernels::kernel(self.bulk_kernel_id); - let mut subtree_cv = if size >= threads && (counter == 0 || (counter & (size as u64 - 1)) == 0) { + let size_u64 = u64::try_from(size).expect("a BLAKE3 subtree size fits in u64"); + let mut subtree_cv = if size >= threads && (counter == 0 || (counter & size_u64.strict_sub(1)) == 0) { const MAX_SUBTREE_CHUNKS: usize = 1 << 12; - let mut subtree_chunks = size / threads; + let mut subtree_chunks = size + .checked_div(threads) + .expect("parallel BLAKE3 thread count is nonzero"); subtree_chunks = subtree_chunks.max(1); subtree_chunks = pow2_floor(subtree_chunks); subtree_chunks = subtree_chunks.min(MAX_SUBTREE_CHUNKS).min(size); - let roots_len = size / subtree_chunks; + let roots_len = size + .checked_div(subtree_chunks) + .expect("parallel BLAKE3 subtree size is nonzero"); debug_assert!(roots_len.is_power_of_two()); - debug_assert_eq!(roots_len * subtree_chunks, size); + debug_assert_eq!(roots_len.strict_mul(subtree_chunks), size); scratch.roots.resize(roots_len, [0u32; 8]); hash_power_of_two_subtree_roots_parallel_rayon(super::SubtreeRootsRequest { @@ -181,7 +187,7 @@ impl Blake3 { ) }; - counter = counter.wrapping_add(size as u64); + counter = counter.strict_add(size_u64); let level = size.trailing_zeros(); let mut total = counter >> level; let mut cv = subtree_cv; @@ -208,12 +214,13 @@ impl Blake3 { } offset_chunks = offset_chunks.strict_add(size); - remaining_commit -= size; + remaining_commit = remaining_commit.strict_sub(size); } - self.cv_stack_len = stack_len as u8; + self.cv_stack_len = u8::try_from(stack_len).expect("BLAKE3 CV stack length fits in u8"); - let new_counter = base_counter.strict_add(batch.batch_chunks as u64); + let batch_chunks = u64::try_from(batch.batch_chunks).expect("a BLAKE3 batch size fits in u64"); + let new_counter = base_counter.strict_add(batch_chunks); self.chunk_state = ChunkState::new( self.key_words, new_counter, @@ -253,12 +260,16 @@ impl Blake3 { let base_counter = self.chunk_state.chunk_counter; let keep_last_full_chunk = input.len().is_multiple_of(CHUNK_LEN) && batch == full_chunks; - let commit = if keep_last_full_chunk { batch - 1 } else { batch }; + let commit = if keep_last_full_chunk { + batch.strict_sub(1) + } else { + batch + }; if commit == 0 { return None; } - let bytes = batch * CHUNK_LEN; + let bytes = batch.strict_mul(CHUNK_LEN); let threads = self.streaming_parallel_threads(bytes, batch, commit)?; if threads <= 1 { return None; @@ -310,9 +321,15 @@ fn compress_parents_parallel_bytes( debug_assert!(out.len() >= child_cvs.len().div_ceil(2)); let pairs = child_cvs.len() / 2; - kernels::parent_cvs_many_from_bytes_inline(kernel.id, &child_cvs[..pairs * 2], key_words, flags, &mut out[..pairs]); + kernels::parent_cvs_many_from_bytes_inline( + kernel.id, + &child_cvs[..pairs.strict_mul(2)], + key_words, + flags, + &mut out[..pairs], + ); if (child_cvs.len() & 1) == 1 { - out[pairs] = child_cvs[child_cvs.len() - 1]; + out[pairs] = child_cvs[child_cvs.len().strict_sub(1)]; pairs.strict_add(1) } else { pairs @@ -334,10 +351,10 @@ fn compress_subtree_wide_bytes( let simd_degree = kernel.id.simd_degree(); let max_leaf_bytes = simd_degree.strict_mul(CHUNK_LEN); if input.len() <= max_leaf_bytes { - let chunks_exact = input.chunks_exact(CHUNK_LEN); - let full_chunks = chunks_exact.len(); + let (chunks, remainder) = input.as_chunks::(); + let full_chunks = chunks.len(); debug_assert!(full_chunks <= simd_degree); - debug_assert!(out.len() >= full_chunks.strict_add(usize::from(!chunks_exact.remainder().is_empty()))); + debug_assert!(out.len() >= full_chunks.strict_add(usize::from(!remainder.is_empty()))); if full_chunks != 0 { // SAFETY: `input` has at least `full_chunks * CHUNK_LEN` bytes and @@ -355,12 +372,12 @@ fn compress_subtree_wide_bytes( } let mut out_len = full_chunks; - let rem = chunks_exact.remainder(); + let rem = remainder; if !rem.is_empty() { let mut cv_words = single_chunk_output( kernel, key_words, - chunk_counter.strict_add(full_chunks as u64), + chunk_counter.strict_add(u64::try_from(full_chunks).expect("a BLAKE3 chunk count fits in u64")), flags, rem, ) @@ -378,7 +395,8 @@ fn compress_subtree_wide_bytes( debug_assert!(out.len() >= simd_degree.max(2)); let left_len = left_subtree_len_bytes(input.len()); let (left, right) = input.split_at(left_len); - let right_chunk_counter = chunk_counter.strict_add((left.len() / CHUNK_LEN) as u64); + let left_chunks = u64::try_from(left.len() / CHUNK_LEN).expect("a BLAKE3 subtree chunk count fits in u64"); + let right_chunk_counter = chunk_counter.strict_add(left_chunks); const MAX_SIMD_DEGREE: usize = 16; let mut cv_array = [[0u8; OUT_LEN]; 2 * MAX_SIMD_DEGREE]; @@ -494,7 +512,7 @@ pub(super) fn root_output_oneshot_join_parallel( debug_assert!(threads > 1); // Cap Rayon recursion depth to approximately match `threads` leaves. - let depth = (usize::BITS - 1 - threads.leading_zeros()) as usize; + let depth = usize::try_from(threads.ilog2()).expect("parallel recursion depth fits in usize"); let budget = depth.max(1); let mut parent_block = @@ -505,7 +523,7 @@ pub(super) fn root_output_oneshot_join_parallel( input_chaining_value: key_words, block_words, counter: 0, - block_len: BLOCK_LEN as u32, + block_len: u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), flags: super::PARENT | flags, }; if flags & (super::KEYED_HASH | super::DERIVE_KEY_MATERIAL) != 0 { diff --git a/src/hashes/crypto/blake3/x86_64.rs b/src/hashes/crypto/blake3/x86_64.rs index 55ee44fd..c6fa176a 100644 --- a/src/hashes/crypto/blake3/x86_64.rs +++ b/src/hashes/crypto/blake3/x86_64.rs @@ -8,13 +8,8 @@ //! //! # Safety //! -//! All functions in this module are marked `unsafe` and require specific CPU -//! features to be present. Callers must verify CPU capabilities before calling. - -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::too_many_arguments)] -#![allow(clippy::many_single_char_names)] +//! Unsafe entry points require specific CPU features to be present. Callers +//! must verify CPU capabilities before calling them. #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; @@ -25,25 +20,55 @@ pub(crate) mod avx2; pub(crate) mod avx512; pub(crate) mod sse41; -use super::{BLOCK_LEN, CHUNK_START, IV, PARENT}; +use super::{BLOCK_LEN, BLOCK_LEN_U32, CHUNK_START, IV, PARENT}; + +/// Inputs shared by the x86 intrinsic hash-many kernels. +pub(crate) struct HashManyRequest<'a, const DEGREE: usize> { + /// One pointer per SIMD lane. + pub(crate) inputs: &'a [*const u8; DEGREE], + /// Complete BLAKE3 blocks in every input. + pub(crate) blocks: usize, + /// Eight-word chaining key. + pub(crate) key: &'a [u32; 8], + /// Initial chunk counter. + pub(crate) counter: u64, + /// Whether to advance the counter for each lane. + pub(crate) increment_counter: bool, + /// Flags applied to every block. + pub(crate) flags: u32, + /// Flags added to the first block. + pub(crate) flags_start: u32, + /// Flags added to the last block. + pub(crate) flags_end: u32, + /// Destination for one chaining value per lane. + pub(crate) out: *mut u8, +} // Shared helpers for SIMD kernels. #[inline(always)] pub(crate) const fn counter_low(counter: u64) -> u32 { - counter as u32 + let [c0, c1, c2, c3, _, _, _, _] = counter.to_le_bytes(); + u32::from_le_bytes([c0, c1, c2, c3]) } #[inline(always)] pub(crate) const fn counter_high(counter: u64) -> u32 { - (counter >> 32) as u32 + let [_, _, _, _, c4, c5, c6, c7] = counter.to_le_bytes(); + u32::from_le_bytes([c4, c5, c6, c7]) } // CV-only compression helpers (avoid `[u32; 16]` materialization) #[cfg(target_arch = "x86_64")] #[inline(always)] -unsafe fn load_msg_vecs(block: *const u8) -> (__m128i, __m128i, __m128i, __m128i) { +/// Loads one unaligned BLAKE3 block into four SIMD registers. +/// +/// # Safety +/// +/// The current CPU must support SSE2, and `block` must be readable for +/// `BLOCK_LEN` bytes. +unsafe fn load_msg_vecs(block: *const u8) -> [__m128i; 4] { // SAFETY: Caller guarantees block pointer is valid for 64 bytes. Intrinsics require SSSE3 via // caller's #[target_feature]. unsafe { @@ -51,12 +76,18 @@ unsafe fn load_msg_vecs(block: *const u8) -> (__m128i, __m128i, __m128i, __m128i let m1 = _mm_loadu_si128(block.add(16).cast()); let m2 = _mm_loadu_si128(block.add(32).cast()); let m3 = _mm_loadu_si128(block.add(48).cast()); - (m0, m1, m2, m3) + [m0, m1, m2, m3] } } #[cfg(target_arch = "x86_64")] #[target_feature(enable = "sse4.1,ssse3")] +/// Compresses one byte-oriented block in place with SSE4.1. +/// +/// # Safety +/// +/// The current CPU must support SSE4.1 and SSSE3, and `block` must be readable +/// for `BLOCK_LEN` bytes. pub(crate) unsafe fn compress_in_place_sse41_bytes( chaining_value: &mut [u32; 8], block: *const u8, @@ -66,8 +97,8 @@ pub(crate) unsafe fn compress_in_place_sse41_bytes( ) { // SAFETY: SSE4.1/SSSE3 intrinsics are available via this function's #[target_feature] attribute. unsafe { - let (m0, m1, m2, m3) = load_msg_vecs(block); - let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + let message = load_msg_vecs(block); + let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, message, counter, block_len, flags); _mm_storeu_si128(chaining_value.as_mut_ptr().cast(), _mm_xor_si128(row0, row2)); _mm_storeu_si128(chaining_value.as_mut_ptr().add(4).cast(), _mm_xor_si128(row1, row3)); } @@ -75,6 +106,12 @@ pub(crate) unsafe fn compress_in_place_sse41_bytes( #[cfg(target_arch = "x86_64")] #[target_feature(enable = "sse4.1,ssse3")] +/// Compresses one byte-oriented block to a chaining value with SSE4.1. +/// +/// # Safety +/// +/// The current CPU must support SSE4.1 and SSSE3, and `block` must be readable +/// for `BLOCK_LEN` bytes. pub(crate) unsafe fn compress_cv_sse41_bytes( chaining_value: &[u32; 8], block: *const u8, @@ -84,8 +121,8 @@ pub(crate) unsafe fn compress_cv_sse41_bytes( ) -> [u32; 8] { // SAFETY: SSE4.1/SSSE3 intrinsics are available via this function's #[target_feature] attribute. unsafe { - let (m0, m1, m2, m3) = load_msg_vecs(block); - let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + let message = load_msg_vecs(block); + let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, message, counter, block_len, flags); let row0 = _mm_xor_si128(row0, row2); let row1 = _mm_xor_si128(row1, row3); @@ -96,6 +133,12 @@ pub(crate) unsafe fn compress_cv_sse41_bytes( } } +/// Compresses one byte-oriented block to a chaining value with AVX2. +/// +/// # Safety +/// +/// The current CPU must support AVX2, SSE4.1, and SSSE3, and `block` must be +/// readable for `BLOCK_LEN` bytes. pub(crate) unsafe fn compress_cv_avx2_bytes( chaining_value: &[u32; 8], block: *const u8, @@ -105,13 +148,19 @@ pub(crate) unsafe fn compress_cv_avx2_bytes( ) -> [u32; 8] { // SAFETY: AVX2/SSE4.1/SSSE3 intrinsics are available via caller's target_feature guarantee. unsafe { - let (m0, m1, m2, m3) = load_msg_vecs(block); - compress_cv_avx2(chaining_value, m0, m1, m2, m3, counter, block_len, flags) + let message = load_msg_vecs(block); + compress_cv_avx2(chaining_value, message, counter, block_len, flags) } } #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2,sse4.1,ssse3")] +/// Compresses one byte-oriented block in place with AVX2. +/// +/// # Safety +/// +/// The current CPU must support AVX2, SSE4.1, and SSSE3, and `block` must be +/// readable for `BLOCK_LEN` bytes. pub(crate) unsafe fn compress_in_place_avx2_bytes( chaining_value: &mut [u32; 8], block: *const u8, @@ -129,11 +178,13 @@ pub(crate) unsafe fn compress_in_place_avx2_bytes( // On ASM-supported platforms, we prefer the handwritten assembly. This intrinsics // version is kept as fallback for other x86_64 platforms (e.g., FreeBSD, illumos). #[cfg(target_arch = "x86_64")] -#[cfg_attr( - any(target_os = "linux", target_os = "macos", target_os = "windows"), - allow(dead_code) -)] #[target_feature(enable = "avx512f,avx512vl,avx2,sse4.1,ssse3")] +/// Compresses one byte-oriented block in place with AVX-512. +/// +/// # Safety +/// +/// The current CPU must support the declared AVX-512, AVX2, SSE4.1, and SSSE3 +/// features, and `block` must be readable for `BLOCK_LEN` bytes. pub(crate) unsafe fn compress_in_place_avx512_bytes( chaining_value: &mut [u32; 8], block: *const u8, @@ -144,19 +195,21 @@ pub(crate) unsafe fn compress_in_place_avx512_bytes( // SAFETY: AVX-512/AVX2/SSE4.1/SSSE3 intrinsics are available via this function's #[target_feature] // attribute. unsafe { - let (m0, m1, m2, m3) = load_msg_vecs(block); - let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + let message = load_msg_vecs(block); + let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, message, counter, block_len, flags); _mm_storeu_si128(chaining_value.as_mut_ptr().cast(), _mm_xor_si128(row0, row2)); _mm_storeu_si128(chaining_value.as_mut_ptr().add(4).cast(), _mm_xor_si128(row1, row3)); } } #[cfg(target_arch = "x86_64")] -#[cfg_attr( - any(target_os = "linux", target_os = "macos", target_os = "windows"), - allow(dead_code) -)] #[target_feature(enable = "avx512f,avx512vl,avx2,sse4.1,ssse3")] +/// Compresses one byte-oriented block to a chaining value with AVX-512. +/// +/// # Safety +/// +/// The current CPU must support the declared AVX-512, AVX2, SSE4.1, and SSSE3 +/// features, and `block` must be readable for `BLOCK_LEN` bytes. pub(crate) unsafe fn compress_cv_avx512_bytes( chaining_value: &[u32; 8], block: *const u8, @@ -167,8 +220,8 @@ pub(crate) unsafe fn compress_cv_avx512_bytes( // SAFETY: AVX-512/AVX2/SSE4.1/SSSE3 intrinsics are available via this function's #[target_feature] // attribute. unsafe { - let (m0, m1, m2, m3) = load_msg_vecs(block); - compress_cv_avx512(chaining_value, m0, m1, m2, m3, counter, block_len, flags) + let message = load_msg_vecs(block); + compress_cv_avx512(chaining_value, message, counter, block_len, flags) } } @@ -182,6 +235,9 @@ pub(crate) unsafe fn compress_cv_avx512_bytes( #[cfg(target_arch = "x86_64")] #[inline(always)] +/// # Safety +/// +/// The current CPU must support SSE2. unsafe fn rot16_sse41(a: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics are available via caller's #[target_feature] attribute. unsafe { _mm_or_si128(_mm_srli_epi32(a, 16), _mm_slli_epi32(a, 16)) } @@ -189,6 +245,9 @@ unsafe fn rot16_sse41(a: __m128i) -> __m128i { #[cfg(target_arch = "x86_64")] #[inline(always)] +/// # Safety +/// +/// The current CPU must support SSE2. unsafe fn rot12_sse41(a: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics are available via caller's #[target_feature] attribute. unsafe { _mm_or_si128(_mm_srli_epi32(a, 12), _mm_slli_epi32(a, 20)) } @@ -196,6 +255,9 @@ unsafe fn rot12_sse41(a: __m128i) -> __m128i { #[cfg(target_arch = "x86_64")] #[inline(always)] +/// # Safety +/// +/// The current CPU must support SSE2. unsafe fn rot8_sse41(a: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics are available via caller's #[target_feature] attribute. unsafe { _mm_or_si128(_mm_srli_epi32(a, 8), _mm_slli_epi32(a, 24)) } @@ -203,6 +265,9 @@ unsafe fn rot8_sse41(a: __m128i) -> __m128i { #[cfg(target_arch = "x86_64")] #[inline(always)] +/// # Safety +/// +/// The current CPU must support SSE2. unsafe fn rot7_sse41(a: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics are available via caller's #[target_feature] attribute. unsafe { _mm_or_si128(_mm_srli_epi32(a, 7), _mm_slli_epi32(a, 25)) } @@ -210,6 +275,11 @@ unsafe fn rot7_sse41(a: __m128i) -> __m128i { #[cfg(target_arch = "x86_64")] #[inline(always)] +/// Executes the first half of the BLAKE3 mixing function. +/// +/// # Safety +/// +/// The current CPU must support SSE4.1 and SSSE3. unsafe fn g1_sse41(row0: &mut __m128i, row1: &mut __m128i, row2: &mut __m128i, row3: &mut __m128i, m: __m128i) { // SAFETY: SSE4.1/SSSE3 intrinsics are available via caller's #[target_feature] attribute. unsafe { @@ -224,6 +294,11 @@ unsafe fn g1_sse41(row0: &mut __m128i, row1: &mut __m128i, row2: &mut __m128i, r #[cfg(target_arch = "x86_64")] #[inline(always)] +/// Executes the second half of the BLAKE3 mixing function. +/// +/// # Safety +/// +/// The current CPU must support SSE4.1 and SSSE3. unsafe fn g2_sse41(row0: &mut __m128i, row1: &mut __m128i, row2: &mut __m128i, row3: &mut __m128i, m: __m128i) { // SAFETY: SSE4.1/SSSE3 intrinsics are available via caller's #[target_feature] attribute. unsafe { @@ -251,6 +326,9 @@ macro_rules! shuffle2 { // Leave row1 unrotated and diagonalize the other rows. #[cfg(target_arch = "x86_64")] #[inline(always)] +/// # Safety +/// +/// The current CPU must support SSE2. unsafe fn diagonalize_sse41(row0: &mut __m128i, row2: &mut __m128i, row3: &mut __m128i) { // SAFETY: SSE2 intrinsics are available via caller's #[target_feature] attribute. unsafe { @@ -262,6 +340,9 @@ unsafe fn diagonalize_sse41(row0: &mut __m128i, row2: &mut __m128i, row3: &mut _ #[cfg(target_arch = "x86_64")] #[inline(always)] +/// # Safety +/// +/// The current CPU must support SSE2. unsafe fn undiagonalize_sse41(row0: &mut __m128i, row2: &mut __m128i, row3: &mut __m128i) { // SAFETY: SSE2 intrinsics are available via caller's #[target_feature] attribute. unsafe { @@ -273,18 +354,21 @@ unsafe fn undiagonalize_sse41(row0: &mut __m128i, row2: &mut __m128i, row3: &mut #[cfg(target_arch = "x86_64")] #[inline(always)] +/// Runs the shared SSE4.1 compression schedule without the output transform. +/// +/// # Safety +/// +/// The current CPU must support SSE4.1 and SSSE3. unsafe fn compress_pre_sse41_impl( chaining_value: &[u32; 8], - mut m0: __m128i, - mut m1: __m128i, - mut m2: __m128i, - mut m3: __m128i, + message: [__m128i; 4], counter: u64, block_len: u32, flags: u32, ) -> [__m128i; 4] { // SAFETY: SSE4.1/SSSE3 intrinsics are available via caller's #[target_feature] attribute. unsafe { + let [mut m0, mut m1, mut m2, mut m3] = message; let mut row0 = _mm_loadu_si128(chaining_value.as_ptr().cast()); let mut row1 = _mm_loadu_si128(chaining_value.as_ptr().add(4).cast()); let mut row2 = _mm_setr_epi32( @@ -294,8 +378,8 @@ unsafe fn compress_pre_sse41_impl( IV[3].cast_signed(), ); let mut row3 = _mm_setr_epi32( - (counter as u32).cast_signed(), - ((counter >> 32) as u32).cast_signed(), + counter_low(counter).cast_signed(), + counter_high(counter).cast_signed(), block_len.cast_signed(), flags.cast_signed(), ); @@ -396,10 +480,7 @@ unsafe fn compress_pre_sse41_impl( #[target_feature(enable = "avx2,sse4.1,ssse3")] unsafe fn compress_cv_avx2( chaining_value: &[u32; 8], - m0: __m128i, - m1: __m128i, - m2: __m128i, - m3: __m128i, + message: [__m128i; 4], counter: u64, block_len: u32, flags: u32, @@ -407,7 +488,7 @@ unsafe fn compress_cv_avx2( // SAFETY: AVX2/SSE4.1/SSSE3 intrinsics are available via this function's #[target_feature] // attribute. unsafe { - let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, message, counter, block_len, flags); let row0 = _mm_xor_si128(row0, row2); let row1 = _mm_xor_si128(row1, row3); let mut out = [0u32; 8]; @@ -425,10 +506,7 @@ unsafe fn compress_cv_avx2( #[target_feature(enable = "avx512f,avx512vl,avx2,sse4.1,ssse3")] unsafe fn compress_cv_avx512( chaining_value: &[u32; 8], - m0: __m128i, - m1: __m128i, - m2: __m128i, - m3: __m128i, + message: [__m128i; 4], counter: u64, block_len: u32, flags: u32, @@ -436,7 +514,7 @@ unsafe fn compress_cv_avx512( // SAFETY: AVX-512/AVX2/SSE4.1/SSSE3 intrinsics are available via this function's #[target_feature] // attribute. unsafe { - let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, message, counter, block_len, flags); let row0 = _mm_xor_si128(row0, row2); let row1 = _mm_xor_si128(row1, row3); let mut out = [0u32; 8]; @@ -472,7 +550,7 @@ pub(crate) unsafe fn compress_avx2( let m2 = _mm_loadu_si128(block_words.as_ptr().add(8).cast()); let m3 = _mm_loadu_si128(block_words.as_ptr().add(12).cast()); let [mut row0, mut row1, mut row2, mut row3] = - compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + compress_pre_sse41_impl(chaining_value, [m0, m1, m2, m3], counter, block_len, flags); let cv_lo = _mm_loadu_si128(chaining_value.as_ptr().cast()); let cv_hi = _mm_loadu_si128(chaining_value.as_ptr().add(4).cast()); @@ -515,7 +593,7 @@ pub(crate) unsafe fn compress_sse41( let m2 = _mm_loadu_si128(block_words.as_ptr().add(8).cast()); let m3 = _mm_loadu_si128(block_words.as_ptr().add(12).cast()); let [mut row0, mut row1, mut row2, mut row3] = - compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + compress_pre_sse41_impl(chaining_value, [m0, m1, m2, m3], counter, block_len, flags); let cv_lo = _mm_loadu_si128(chaining_value.as_ptr().cast()); let cv_hi = _mm_loadu_si128(chaining_value.as_ptr().add(4).cast()); @@ -558,10 +636,10 @@ pub(crate) unsafe fn chunk_compress_blocks_sse41( chaining_value, blocks.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); return; } @@ -573,10 +651,10 @@ pub(crate) unsafe fn chunk_compress_blocks_sse41( chaining_value, block_bytes.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } } @@ -600,7 +678,7 @@ pub(crate) unsafe fn parent_cv_sse41( let m2 = _mm_loadu_si128(right_child_cv.as_ptr().cast()); let m3 = _mm_loadu_si128(right_child_cv.as_ptr().add(4).cast()); let [row0, row1, row2, row3] = - compress_pre_sse41_impl(&key_words, m0, m1, m2, m3, 0, BLOCK_LEN as u32, PARENT | flags); + compress_pre_sse41_impl(&key_words, [m0, m1, m2, m3], 0, BLOCK_LEN_U32, PARENT | flags); let row0 = _mm_xor_si128(row0, row2); let row1 = _mm_xor_si128(row1, row3); let mut out = [0u32; 8]; @@ -636,10 +714,10 @@ pub(crate) unsafe fn chunk_compress_blocks_avx2( chaining_value, blocks.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); return; } @@ -651,10 +729,10 @@ pub(crate) unsafe fn chunk_compress_blocks_avx2( chaining_value, block_bytes.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } } @@ -678,7 +756,7 @@ pub(crate) unsafe fn parent_cv_avx2( let m1 = _mm_loadu_si128(left_child_cv.as_ptr().add(4).cast()); let m2 = _mm_loadu_si128(right_child_cv.as_ptr().cast()); let m3 = _mm_loadu_si128(right_child_cv.as_ptr().add(4).cast()); - compress_cv_avx2(&key_words, m0, m1, m2, m3, 0, BLOCK_LEN as u32, PARENT | flags) + compress_cv_avx2(&key_words, [m0, m1, m2, m3], 0, BLOCK_LEN_U32, PARENT | flags) } } @@ -730,7 +808,7 @@ pub(crate) unsafe fn chunk_compress_blocks_avx512( chaining_value, blocks.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); } @@ -740,11 +818,11 @@ pub(crate) unsafe fn chunk_compress_blocks_avx512( chaining_value, blocks.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); } - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); return; } @@ -758,7 +836,7 @@ pub(crate) unsafe fn chunk_compress_blocks_avx512( chaining_value, block_bytes.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); } @@ -768,11 +846,11 @@ pub(crate) unsafe fn chunk_compress_blocks_avx512( chaining_value, block_bytes.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); } - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } } @@ -796,6 +874,6 @@ pub(crate) unsafe fn parent_cv_avx512( let m1 = _mm_loadu_si128(left_child_cv.as_ptr().add(4).cast()); let m2 = _mm_loadu_si128(right_child_cv.as_ptr().cast()); let m3 = _mm_loadu_si128(right_child_cv.as_ptr().add(4).cast()); - compress_cv_avx512(&key_words, m0, m1, m2, m3, 0, BLOCK_LEN as u32, PARENT | flags) + compress_cv_avx512(&key_words, [m0, m1, m2, m3], 0, BLOCK_LEN_U32, PARENT | flags) } } diff --git a/src/hashes/crypto/blake3/x86_64/asm.rs b/src/hashes/crypto/blake3/x86_64/asm.rs index 7b58649d..c29a4a34 100644 --- a/src/hashes/crypto/blake3/x86_64/asm.rs +++ b/src/hashes/crypto/blake3/x86_64/asm.rs @@ -8,8 +8,6 @@ //! preprocessor conditionals removed and symbols renamed. //! - The implementation is compiled via `global_asm!` (no external objects). -#![allow(unsafe_code)] - use core::arch::global_asm; #[cfg(target_os = "linux")] @@ -33,7 +31,7 @@ global_asm!(include_str!("asm/rscrypto_blake3_avx512_x86-64_windows_msvc.s")); // entrypoints are internal to rscrypto and are only called from our own code. #[cfg(target_os = "windows")] unsafe extern "sysv64" { - pub fn rscrypto_blake3_hash_many_avx2( + fn rscrypto_blake3_hash_many_avx2( inputs: *const *const u8, num_inputs: usize, blocks: usize, @@ -46,7 +44,7 @@ unsafe extern "sysv64" { out: *mut u8, ); - pub fn rscrypto_blake3_hash_many_avx512( + fn rscrypto_blake3_hash_many_avx512( inputs: *const *const u8, num_inputs: usize, blocks: usize, @@ -59,7 +57,7 @@ unsafe extern "sysv64" { out: *mut u8, ); - pub fn rscrypto_blake3_xof_many_avx512( + fn rscrypto_blake3_xof_many_avx512( cv: *const u32, block: *const u8, block_len: u8, @@ -69,19 +67,13 @@ unsafe extern "sysv64" { outblocks: usize, ); - pub fn rscrypto_blake3_compress_in_place_avx512( - cv: *mut u32, - block: *const u8, - counter: u64, - block_len: u8, - flags: u8, - ); + fn rscrypto_blake3_compress_in_place_avx512(cv: *mut u32, block: *const u8, counter: u64, block_len: u8, flags: u8); } #[cfg(not(target_os = "windows"))] unsafe extern "C" { - pub fn rscrypto_blake3_hash_many_avx2( + fn rscrypto_blake3_hash_many_avx2( inputs: *const *const u8, num_inputs: usize, blocks: usize, @@ -94,7 +86,7 @@ unsafe extern "C" { out: *mut u8, ); - pub fn rscrypto_blake3_hash_many_avx512( + fn rscrypto_blake3_hash_many_avx512( inputs: *const *const u8, num_inputs: usize, blocks: usize, @@ -107,7 +99,7 @@ unsafe extern "C" { out: *mut u8, ); - pub fn rscrypto_blake3_xof_many_avx512( + fn rscrypto_blake3_xof_many_avx512( cv: *const u32, block: *const u8, block_len: u8, @@ -117,16 +109,34 @@ unsafe extern "C" { outblocks: usize, ); - pub fn rscrypto_blake3_compress_in_place_avx512( - cv: *mut u32, - block: *const u8, - counter: u64, - block_len: u8, - flags: u8, - ); + fn rscrypto_blake3_compress_in_place_avx512(cv: *mut u32, block: *const u8, counter: u64, block_len: u8, flags: u8); } +/// Raw parameters for an x86 BLAKE3 hash-many assembly call. +pub(crate) struct HashManyRequest { + /// Input pointer array. + pub(crate) inputs: *const *const u8, + /// Number of input pointers. + pub(crate) num_inputs: usize, + /// Complete BLAKE3 blocks in each input. + pub(crate) blocks: usize, + /// Eight-word chaining key. + pub(crate) key: *const u32, + /// Initial chunk counter. + pub(crate) counter: u64, + /// Whether to advance the counter for each input. + pub(crate) increment_counter: bool, + /// Flags applied to every block. + pub(crate) flags: u8, + /// Flags added to the first block. + pub(crate) flags_start: u8, + /// Flags added to the last block. + pub(crate) flags_end: u8, + /// Destination for one chaining value per input. + pub(crate) out: *mut u8, +} + /// AVX2 `hash_many` assembly entrypoint. /// /// # Safety @@ -140,18 +150,19 @@ unsafe extern "C" { /// 6. The output range does not alias any input range or `key`. /// 7. `num_inputs`, `blocks`, counters, and flags are public values. #[inline(always)] -pub(crate) unsafe fn hash_many_avx2( - inputs: *const *const u8, - num_inputs: usize, - blocks: usize, - key: *const u32, - counter: u64, - increment_counter: bool, - flags: u8, - flags_start: u8, - flags_end: u8, - out: *mut u8, -) { +pub(crate) unsafe fn hash_many_avx2(request: HashManyRequest) { + let HashManyRequest { + inputs, + num_inputs, + blocks, + key, + counter, + increment_counter, + flags, + flags_start, + flags_end, + out, + } = request; // SAFETY: AVX2 `hash_many` FFI call because: // 1. The caller upholds this wrapper's CPU-feature contract. // 2. The caller upholds the input pointer, key, output, and aliasing contracts. @@ -185,18 +196,19 @@ pub(crate) unsafe fn hash_many_avx2( /// 6. The output range does not alias any input range or `key`. /// 7. `num_inputs`, `blocks`, counters, and flags are public values. #[inline(always)] -pub(crate) unsafe fn hash_many_avx512( - inputs: *const *const u8, - num_inputs: usize, - blocks: usize, - key: *const u32, - counter: u64, - increment_counter: bool, - flags: u8, - flags_start: u8, - flags_end: u8, - out: *mut u8, -) { +pub(crate) unsafe fn hash_many_avx512(request: HashManyRequest) { + let HashManyRequest { + inputs, + num_inputs, + blocks, + key, + counter, + increment_counter, + flags, + flags_start, + flags_end, + out, + } = request; // SAFETY: AVX-512 `hash_many` FFI call because: // 1. The caller upholds this wrapper's CPU-feature contract. // 2. The caller upholds the input pointer, key, output, and aliasing contracts. @@ -265,8 +277,8 @@ pub(crate) unsafe fn compress_in_place_avx512( block_len: u32, flags: u32, ) -> [u32; 8] { - debug_assert!(block_len <= u8::MAX as u32); - debug_assert!(flags <= u8::MAX as u32); + let block_len = u8::try_from(block_len).expect("BLAKE3 block length fits the assembly ABI"); + let flags = u8::try_from(flags).expect("BLAKE3 flags fit the assembly ABI"); let mut cv_out = *cv; // SAFETY: AVX-512 single-block FFI call because: // 1. The caller upholds this wrapper's CPU-feature contract. @@ -274,7 +286,7 @@ pub(crate) unsafe fn compress_in_place_avx512( // 3. The caller guarantees `block` is readable for one BLAKE3 block. // 4. Debug assertions document the ABI narrowing for `block_len` and `flags`. unsafe { - rscrypto_blake3_compress_in_place_avx512(cv_out.as_mut_ptr(), block, counter, block_len as u8, flags as u8); + rscrypto_blake3_compress_in_place_avx512(cv_out.as_mut_ptr(), block, counter, block_len, flags); } cv_out } @@ -297,14 +309,14 @@ pub(crate) unsafe fn compress_in_place_avx512_mut( block_len: u32, flags: u32, ) { - debug_assert!(block_len <= u8::MAX as u32); - debug_assert!(flags <= u8::MAX as u32); + let block_len = u8::try_from(block_len).expect("BLAKE3 block length fits the assembly ABI"); + let flags = u8::try_from(flags).expect("BLAKE3 flags fit the assembly ABI"); // SAFETY: AVX-512 in-place single-block FFI call because: // 1. The caller upholds this wrapper's CPU-feature contract. // 2. `cv` is writable for 8 words and the mutable borrow prevents another Rust alias. // 3. The caller guarantees `block` is readable for one BLAKE3 block and does not alias `cv`. // 4. Debug assertions document the ABI narrowing for `block_len` and `flags`. unsafe { - rscrypto_blake3_compress_in_place_avx512(cv.as_mut_ptr(), block, counter, block_len as u8, flags as u8); + rscrypto_blake3_compress_in_place_avx512(cv.as_mut_ptr(), block, counter, block_len, flags); } } diff --git a/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx2_x86-64_apple_darwin.s b/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx2_x86-64_apple_darwin.s index 80cfa159..1d01f25e 100644 --- a/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx2_x86-64_apple_darwin.s +++ b/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx2_x86-64_apple_darwin.s @@ -5,7 +5,6 @@ // Symbols are renamed into the rscrypto namespace and the Darwin-specific // symbol spelling is kept explicit for Rust global_asm!. -.intel_syntax noprefix diff --git a/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx512_x86-64_apple_darwin.s b/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx512_x86-64_apple_darwin.s index 00177327..449a5f91 100644 --- a/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx512_x86-64_apple_darwin.s +++ b/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx512_x86-64_apple_darwin.s @@ -5,7 +5,6 @@ // Symbols are renamed into the rscrypto namespace and the Darwin-specific // symbol spelling is kept explicit for Rust global_asm!. -.intel_syntax noprefix diff --git a/src/hashes/crypto/blake3/x86_64/avx2.rs b/src/hashes/crypto/blake3/x86_64/avx2.rs index e0fb67c6..4401abc3 100644 --- a/src/hashes/crypto/blake3/x86_64/avx2.rs +++ b/src/hashes/crypto/blake3/x86_64/avx2.rs @@ -1,26 +1,23 @@ //! BLAKE3 x86_64 AVX2 throughput kernel (8-way). -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::too_many_arguments)] -#![allow(clippy::many_single_char_names)] // Supported OS builds still keep the assembly ABI around for sub-degree tails // and parent reductions. The owned intrinsic body is also used directly for // full contiguous batches and by diagnostic benches. -#![cfg_attr( - any(target_os = "linux", target_os = "macos", target_os = "windows"), - allow(dead_code, unused_imports) -)] use core::arch::x86_64::*; +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] +use super::super::{CHUNK_END, CHUNK_LEN, CHUNK_START, OUT_LEN, PARENT}; use super::{ - super::{BLOCK_LEN, CHUNK_END, CHUNK_LEN, CHUNK_START, IV, MSG_SCHEDULE, OUT_LEN, PARENT}, - counter_high, counter_low, + super::{BLOCK_LEN, BLOCK_LEN_U32, IV, MSG_SCHEDULE}, + HashManyRequest, counter_high, counter_low, }; -pub const DEGREE: usize = 8; +pub(crate) const DEGREE: usize = 8; +/// # Safety +/// +/// AVX2 must be available and `src` must be readable for 32 bytes. #[inline(always)] unsafe fn loadu(src: *const u8) -> __m256i { // SAFETY: Unaligned AVX2 load because: @@ -30,6 +27,9 @@ unsafe fn loadu(src: *const u8) -> __m256i { unsafe { _mm256_loadu_si256(src.cast()) } } +/// # Safety +/// +/// AVX2 must be available and `dest` must be writable for 32 bytes. #[inline(always)] unsafe fn storeu(src: __m256i, dest: *mut u8) { // SAFETY: Unaligned AVX2 store because: @@ -39,6 +39,9 @@ unsafe fn storeu(src: __m256i, dest: *mut u8) { unsafe { _mm256_storeu_si256(dest.cast(), src) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn add(a: __m256i, b: __m256i) -> __m256i { // SAFETY: AVX2 lane add because: @@ -47,6 +50,9 @@ unsafe fn add(a: __m256i, b: __m256i) -> __m256i { unsafe { _mm256_add_epi32(a, b) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn xor(a: __m256i, b: __m256i) -> __m256i { // SAFETY: AVX2 lane xor because: @@ -55,6 +61,9 @@ unsafe fn xor(a: __m256i, b: __m256i) -> __m256i { unsafe { _mm256_xor_si256(a, b) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn set1(x: u32) -> __m256i { // SAFETY: AVX2 scalar broadcast because: @@ -63,25 +72,31 @@ unsafe fn set1(x: u32) -> __m256i { unsafe { _mm256_set1_epi32(x.cast_signed()) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] -unsafe fn set8(a: u32, b: u32, c: u32, d: u32, e: u32, f: u32, g: u32, h: u32) -> __m256i { +unsafe fn set8(lanes: [u32; DEGREE]) -> __m256i { // SAFETY: AVX2 lane construction because: // 1. Any `u32` bit pattern is valid lane data after reinterpretation as `i32`. // 2. The caller only reaches this helper with AVX2 available. unsafe { _mm256_setr_epi32( - a.cast_signed(), - b.cast_signed(), - c.cast_signed(), - d.cast_signed(), - e.cast_signed(), - f.cast_signed(), - g.cast_signed(), - h.cast_signed(), + lanes[0].cast_signed(), + lanes[1].cast_signed(), + lanes[2].cast_signed(), + lanes[3].cast_signed(), + lanes[4].cast_signed(), + lanes[5].cast_signed(), + lanes[6].cast_signed(), + lanes[7].cast_signed(), ) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn rot12(x: __m256i) -> __m256i { // SAFETY: AVX2 rotate-right-by-12 sequence because: @@ -90,6 +105,9 @@ unsafe fn rot12(x: __m256i) -> __m256i { unsafe { _mm256_or_si256(_mm256_srli_epi32(x, 12), _mm256_slli_epi32(x, 20)) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn rot7(x: __m256i) -> __m256i { // SAFETY: AVX2 rotate-right-by-7 sequence because: @@ -98,6 +116,9 @@ unsafe fn rot7(x: __m256i) -> __m256i { unsafe { _mm256_or_si256(_mm256_srli_epi32(x, 7), _mm256_slli_epi32(x, 25)) } } +/// # Safety +/// +/// AVX2 must be available and `r` must be in `0..7`. #[inline(always)] unsafe fn round(v: &mut [__m256i; 16], m: &[__m256i; 16], r: usize, rot16_mask: __m256i, rot8_mask: __m256i) { // SAFETY: One AVX2 BLAKE3 round because: @@ -223,6 +244,9 @@ unsafe fn round(v: &mut [__m256i; 16], m: &[__m256i; 16], r: usize, rot16_mask: } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn interleave128(a: __m256i, b: __m256i) -> (__m256i, __m256i) { // SAFETY: AVX2 128-bit lane interleave because: @@ -237,6 +261,9 @@ unsafe fn interleave128(a: __m256i, b: __m256i) -> (__m256i, __m256i) { } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] pub(super) unsafe fn transpose8x8(vecs: &mut [__m256i; 8]) { // SAFETY: AVX2 8x8 register transpose because: @@ -278,6 +305,10 @@ pub(super) unsafe fn transpose8x8(vecs: &mut [__m256i; 8]) { } } +/// # Safety +/// +/// AVX2 must be available and every input must be readable for a complete +/// block starting at `block_offset`. #[inline(always)] unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) -> [__m256i; 16] { // SAFETY: AVX2 message transpose because: @@ -287,7 +318,7 @@ unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) // dereferenced by Rust. // 4. The transpose operates only on local vector registers. unsafe { - let stride = 4 * DEGREE; + let stride = 4usize.strict_mul(DEGREE); let mut half0 = [ loadu(inputs[0].add(block_offset)), loadu(inputs[1].add(block_offset)), @@ -299,18 +330,21 @@ unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) loadu(inputs[7].add(block_offset)), ]; let mut half1 = [ - loadu(inputs[0].add(block_offset + stride)), - loadu(inputs[1].add(block_offset + stride)), - loadu(inputs[2].add(block_offset + stride)), - loadu(inputs[3].add(block_offset + stride)), - loadu(inputs[4].add(block_offset + stride)), - loadu(inputs[5].add(block_offset + stride)), - loadu(inputs[6].add(block_offset + stride)), - loadu(inputs[7].add(block_offset + stride)), + loadu(inputs[0].add(block_offset.strict_add(stride))), + loadu(inputs[1].add(block_offset.strict_add(stride))), + loadu(inputs[2].add(block_offset.strict_add(stride))), + loadu(inputs[3].add(block_offset.strict_add(stride))), + loadu(inputs[4].add(block_offset.strict_add(stride))), + loadu(inputs[5].add(block_offset.strict_add(stride))), + loadu(inputs[6].add(block_offset.strict_add(stride))), + loadu(inputs[7].add(block_offset.strict_add(stride))), ]; for &input in inputs.iter() { - _mm_prefetch(input.wrapping_add(block_offset + 256).cast::(), _MM_HINT_T0); + _mm_prefetch( + input.wrapping_add(block_offset.strict_add(256)).cast::(), + _MM_HINT_T0, + ); } transpose8x8(&mut half0); @@ -323,6 +357,9 @@ unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn load_counters(counter: u64, increment_counter: bool) -> (__m256i, __m256i) { let mask = if increment_counter { !0u64 } else { 0u64 }; @@ -332,7 +369,7 @@ unsafe fn load_counters(counter: u64, increment_counter: bool) -> (__m256i, __m2 // 3. The caller only reaches this helper with AVX2 available. unsafe { ( - set8( + set8([ counter_low(counter), counter_low(counter.wrapping_add(mask & 1)), counter_low(counter.wrapping_add(mask & 2)), @@ -341,8 +378,8 @@ unsafe fn load_counters(counter: u64, increment_counter: bool) -> (__m256i, __m2 counter_low(counter.wrapping_add(mask & 5)), counter_low(counter.wrapping_add(mask & 6)), counter_low(counter.wrapping_add(mask & 7)), - ), - set8( + ]), + set8([ counter_high(counter), counter_high(counter.wrapping_add(mask & 1)), counter_high(counter.wrapping_add(mask & 2)), @@ -351,23 +388,30 @@ unsafe fn load_counters(counter: u64, increment_counter: bool) -> (__m256i, __m2 counter_high(counter.wrapping_add(mask & 5)), counter_high(counter.wrapping_add(mask & 6)), counter_high(counter.wrapping_add(mask & 7)), - ), + ]), ) } } +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] macro_rules! avx2_shuffle { ($z:expr, $y:expr, $x:expr, $w:expr) => { ($z << 6) | ($y << 4) | ($x << 2) | $w }; } +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] macro_rules! shuffle2 { ($a:expr, $b:expr, $c:expr) => { _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps($a), _mm256_castsi256_ps($b), $c)) }; } +/// # Safety +/// +/// AVX2 must be available, and both pointers must be readable for 16 bytes +/// starting at `offset`. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn load2x128(lo: *const u8, hi: *const u8, offset: usize) -> __m256i { // SAFETY: Loading two 128-bit halves into one YMM register because: @@ -381,6 +425,10 @@ unsafe fn load2x128(lo: *const u8, hi: *const u8, offset: usize) -> __m256i { } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn set2x128(row: __m128i) -> __m256i { // SAFETY: Duplicating one 128-bit row into both halves because: @@ -389,6 +437,10 @@ unsafe fn set2x128(row: __m128i) -> __m256i { unsafe { _mm256_broadcastsi128_si256(row) } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn g1_2( row0: &mut __m256i, @@ -412,6 +464,10 @@ unsafe fn g1_2( } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn g2_2( row0: &mut __m256i, @@ -435,6 +491,10 @@ unsafe fn g2_2( } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn diagonalize_2(row0: &mut __m256i, row2: &mut __m256i, row3: &mut __m256i) { // SAFETY: Two-lane AVX2 diagonalization because: @@ -447,6 +507,10 @@ unsafe fn diagonalize_2(row0: &mut __m256i, row2: &mut __m256i, row3: &mut __m25 } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn undiagonalize_2(row0: &mut __m256i, row2: &mut __m256i, row3: &mut __m256i) { // SAFETY: Two-lane AVX2 undiagonalization because: @@ -459,16 +523,14 @@ unsafe fn undiagonalize_2(row0: &mut __m256i, row2: &mut __m256i, row3: &mut __m } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn compress2_pre( - mut row0: __m256i, - mut row1: __m256i, - mut row2: __m256i, - mut row3: __m256i, - mut m0: __m256i, - mut m1: __m256i, - mut m2: __m256i, - mut m3: __m256i, + [mut row0, mut row1, mut row2, mut row3]: [__m256i; 4], + [mut m0, mut m1, mut m2, mut m3]: [__m256i; 4], ) -> [__m256i; 4] { // SAFETY: Two-lane AVX2 compression preimage because: // 1. The caller only reaches this helper with AVX2 available. @@ -565,6 +627,10 @@ unsafe fn compress2_pre( } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn iv_row2x128() -> __m256i { // SAFETY: Duplicating the fixed BLAKE3 IV row because: @@ -580,6 +646,10 @@ unsafe fn iv_row2x128() -> __m256i { } } +/// # Safety +/// +/// AVX2 must be available and `out` must be writable for two chaining values. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn store2_cvs(row0: __m256i, row1: __m256i, out: *mut u8) { // SAFETY: Storing two 32-byte CVs from two independent 128-bit lanes because: @@ -590,10 +660,17 @@ unsafe fn store2_cvs(row0: __m256i, row1: __m256i, out: *mut u8) { _mm_storeu_si128(out.cast(), _mm256_castsi256_si128(row0)); _mm_storeu_si128(out.add(16).cast(), _mm256_castsi256_si128(row1)); _mm_storeu_si128(out.add(OUT_LEN).cast(), _mm256_extracti128_si256(row0, 1)); - _mm_storeu_si128(out.add(OUT_LEN + 16).cast(), _mm256_extracti128_si256(row1, 1)); + _mm_storeu_si128( + out.add(OUT_LEN.strict_add(16)).cast(), + _mm256_extracti128_si256(row1, 1), + ); } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn compress2_parent_pre( key: &[u32; 8], @@ -609,19 +686,18 @@ unsafe fn compress2_parent_pre( // 3. `m0..m3` are the two 64-byte parent blocks. unsafe { compress2_pre( - set2x128(_mm_loadu_si128(key.as_ptr().cast())), - set2x128(_mm_loadu_si128(key.as_ptr().add(4).cast())), - iv_row2x128(), - set2x128(_mm_setr_epi32( - 0, - 0, - (BLOCK_LEN as u32).cast_signed(), - (PARENT | flags).cast_signed(), - )), - m0, - m1, - m2, - m3, + [ + set2x128(_mm_loadu_si128(key.as_ptr().cast())), + set2x128(_mm_loadu_si128(key.as_ptr().add(4).cast())), + iv_row2x128(), + set2x128(_mm_setr_epi32( + 0, + 0, + BLOCK_LEN_U32.cast_signed(), + (PARENT | flags).cast_signed(), + )), + ], + [m0, m1, m2, m3], ) } } @@ -634,6 +710,7 @@ unsafe fn compress2_parent_pre( /// 1. AVX2 is available on the current CPU. /// 2. `parents[0]` and `parents[1]` are each readable for one 64-byte parent block. /// 3. `out` is writable for two 32-byte CV outputs. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[target_feature(enable = "avx2")] pub(crate) unsafe fn parent_cv2_owned(parents: &[*const u8; 2], key: &[u32; 8], flags: u32, out: *mut u8) { // SAFETY: Two-parent AVX2 CV reduction because: @@ -663,6 +740,7 @@ pub(crate) unsafe fn parent_cv2_owned(parents: &[*const u8; 2], key: &[u32; 8], /// 1. AVX2 is available on the current CPU. /// 2. `input` is readable for two full BLAKE3 chunks. /// 3. `out` is writable for two 32-byte CV outputs. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[target_feature(enable = "avx2")] pub(crate) unsafe fn hash2_chunks_owned(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { // SAFETY: Two-chunk AVX2 CV reduction because: @@ -680,14 +758,14 @@ pub(crate) unsafe fn hash2_chunks_owned(input: *const u8, key: &[u32; 8], counte let counter1 = counter.wrapping_add(1); let counter1_low = counter_low(counter1).cast_signed(); let counter1_high = counter_high(counter1).cast_signed(); - let block_len = (BLOCK_LEN as u32).cast_signed(); + let block_len = BLOCK_LEN_U32.cast_signed(); for block_idx in 0..(CHUNK_LEN / BLOCK_LEN) { let mut block_flags = flags; if block_idx == 0 { block_flags |= CHUNK_START; } - if block_idx + 1 == CHUNK_LEN / BLOCK_LEN { + if block_idx.strict_add(1) == CHUNK_LEN / BLOCK_LEN { block_flags |= CHUNK_END; } @@ -702,16 +780,15 @@ pub(crate) unsafe fn hash2_chunks_owned(input: *const u8, key: &[u32; 8], counte _mm_setr_epi32(counter1_low, counter1_high, block_len, block_flags.cast_signed()), 1, ); - let offset = block_idx * BLOCK_LEN; + let offset = block_idx.strict_mul(BLOCK_LEN); let [mut v0, mut v1, v2, v3] = compress2_pre( - row0, - row1, - row2, - row3, - load2x128(input, input1, offset), - load2x128(input, input1, offset + 16), - load2x128(input, input1, offset + 32), - load2x128(input, input1, offset + 48), + [row0, row1, row2, row3], + [ + load2x128(input, input1, offset), + load2x128(input, input1, offset.strict_add(16)), + load2x128(input, input1, offset.strict_add(32)), + load2x128(input, input1, offset.strict_add(48)), + ], ); v0 = xor(v0, v2); v1 = xor(v1, v3); @@ -735,15 +812,17 @@ pub(crate) unsafe fn hash2_chunks_owned(input: *const u8, key: &[u32; 8], counte /// `DEGREE * OUT_LEN` writable bytes. #[target_feature(enable = "avx2")] pub(crate) unsafe fn hash8_owned( - inputs: &[*const u8; DEGREE], - blocks: usize, - key: &[u32; 8], - counter: u64, - increment_counter: bool, - flags: u32, - flags_start: u32, - flags_end: u32, - out: *mut u8, + HashManyRequest { + inputs, + blocks, + key, + counter, + increment_counter, + flags, + flags_start, + flags_end, + out, + }: HashManyRequest<'_, DEGREE>, ) { // SAFETY: 8-way AVX2 BLAKE3 hash-many because: // 1. The caller guarantees AVX2 availability for this target-feature function. @@ -758,7 +837,7 @@ pub(crate) unsafe fn hash8_owned( 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12, 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12, ); - let block_len_vec = set1(BLOCK_LEN as u32); + let block_len_vec = set1(BLOCK_LEN_U32); let iv0 = set1(IV[0]); let iv1 = set1(IV[1]); let iv2 = set1(IV[2]); @@ -782,12 +861,12 @@ pub(crate) unsafe fn hash8_owned( if block == 0 { block_flags |= flags_start; } - if block + 1 == blocks { + if block.strict_add(1) == blocks { block_flags |= flags_end; } let block_flags_vec = set1(block_flags); - let msg_vecs = transpose_msg_vecs(inputs, block * BLOCK_LEN); + let msg_vecs = transpose_msg_vecs(inputs, block.strict_mul(BLOCK_LEN)); let mut v = [ h_vecs[0], @@ -829,15 +908,15 @@ pub(crate) unsafe fn hash8_owned( // Unlike SSE4.1, this transpose yields output vecs already ordered by word. transpose8x8(&mut h_vecs); - let stride = 4 * DEGREE; + let stride = 4usize.strict_mul(DEGREE); storeu(h_vecs[0], out); storeu(h_vecs[1], out.add(stride)); - storeu(h_vecs[2], out.add(2 * stride)); - storeu(h_vecs[3], out.add(3 * stride)); - storeu(h_vecs[4], out.add(4 * stride)); - storeu(h_vecs[5], out.add(5 * stride)); - storeu(h_vecs[6], out.add(6 * stride)); - storeu(h_vecs[7], out.add(7 * stride)); + storeu(h_vecs[2], out.add(2usize.strict_mul(stride))); + storeu(h_vecs[3], out.add(3usize.strict_mul(stride))); + storeu(h_vecs[4], out.add(4usize.strict_mul(stride))); + storeu(h_vecs[5], out.add(5usize.strict_mul(stride))); + storeu(h_vecs[6], out.add(6usize.strict_mul(stride))); + storeu(h_vecs[7], out.add(7usize.strict_mul(stride))); } } @@ -849,33 +928,11 @@ pub(crate) unsafe fn hash8_owned( /// for `blocks * BLOCK_LEN` bytes. #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] #[target_feature(enable = "avx2")] -pub(crate) unsafe fn hash8( - inputs: &[*const u8; DEGREE], - blocks: usize, - key: &[u32; 8], - counter: u64, - increment_counter: bool, - flags: u32, - flags_start: u32, - flags_end: u32, - out: *mut u8, -) { +pub(crate) unsafe fn hash8(request: HashManyRequest<'_, DEGREE>) { // SAFETY: Forwarding to the owned AVX2 implementation because: // 1. This function has the same AVX2 target-feature requirement. // 2. The caller's pointer/output contract is identical to `hash8_owned`. - unsafe { - hash8_owned( - inputs, - blocks, - key, - counter, - increment_counter, - flags, - flags_start, - flags_end, - out, - ) - } + unsafe { hash8_owned(request) } } /// Generate 8 root output blocks (64 bytes each) in parallel. @@ -999,7 +1056,7 @@ pub(crate) unsafe fn root_output_blocks8( transpose8x8(&mut out_hi); for lane in 0..DEGREE { - let base = out.add(lane * 64); + let base = out.add(lane.strict_mul(64)); storeu(out_lo[lane], base); storeu(out_hi[lane], base.add(32)); } diff --git a/src/hashes/crypto/blake3/x86_64/avx512.rs b/src/hashes/crypto/blake3/x86_64/avx512.rs index 8388d138..aee01992 100644 --- a/src/hashes/crypto/blake3/x86_64/avx512.rs +++ b/src/hashes/crypto/blake3/x86_64/avx512.rs @@ -1,57 +1,64 @@ //! BLAKE3 x86_64 AVX-512 throughput kernel (16-way). -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::too_many_arguments)] -#![allow(clippy::many_single_char_names)] -// On Linux we currently prefer the upstream asm implementation; keep the -// intrinsic fallback compiled but don't let `-D warnings` turn it into a build -// failure. -#![cfg_attr( - any(target_os = "linux", target_os = "macos", target_os = "windows"), - allow(dead_code, unused_imports) -)] - use core::arch::x86_64::*; use super::{ - super::{BLOCK_LEN, CHUNK_LEN, IV, MSG_SCHEDULE, OUT_LEN}, + super::{BLOCK_LEN, BLOCK_LEN_U32, CHUNK_LEN, IV, MSG_SCHEDULE, OUT_LEN}, + HashManyRequest, avx2::transpose8x8, counter_high, counter_low, }; -pub const DEGREE: usize = 16; +pub(crate) const DEGREE: usize = 16; +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn add(a: __m512i, b: __m512i) -> __m512i { // SAFETY: Caller guarantees the required AVX-512 feature set for this backend. unsafe { _mm512_add_epi32(a, b) } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn xor(a: __m512i, b: __m512i) -> __m512i { // SAFETY: Caller guarantees the required AVX-512 feature set for this backend. unsafe { _mm512_xor_si512(a, b) } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn set1(x: u32) -> __m512i { // SAFETY: Caller guarantees the required AVX-512 feature set for this backend. unsafe { _mm512_set1_epi32(x.cast_signed()) } } +/// # Safety +/// +/// AVX2 must be available and `src` must be readable for 32 bytes. #[inline(always)] unsafe fn loadu256(src: *const u8) -> __m256i { // SAFETY: Caller guarantees `src` is valid to read 32 bytes and has enabled this AVX-512 backend. unsafe { _mm256_loadu_si256(src.cast()) } } +/// # Safety +/// +/// AVX2 must be available and `dest` must be writable for 32 bytes. #[inline(always)] unsafe fn storeu256(src: __m256i, dest: *mut u8) { // SAFETY: Caller guarantees `dest` is valid to write 32 bytes and has enabled this AVX-512 backend. unsafe { _mm256_storeu_si256(dest.cast(), src) } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn rot16(x: __m512i) -> __m512i { // 32-bit rotate by 16. Prefer `vprold` over shift/or. @@ -59,6 +66,9 @@ unsafe fn rot16(x: __m512i) -> __m512i { unsafe { _mm512_rol_epi32(x, 16) } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn rot12(x: __m512i) -> __m512i { // Rotate right by 12 == rotate left by 20. @@ -66,6 +76,9 @@ unsafe fn rot12(x: __m512i) -> __m512i { unsafe { _mm512_rol_epi32(x, 20) } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn rot8(x: __m512i) -> __m512i { // Rotate right by 8 == rotate left by 24. @@ -73,6 +86,9 @@ unsafe fn rot8(x: __m512i) -> __m512i { unsafe { _mm512_rol_epi32(x, 24) } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn rot7(x: __m512i) -> __m512i { // Rotate right by 7 == rotate left by 25. @@ -80,6 +96,9 @@ unsafe fn rot7(x: __m512i) -> __m512i { unsafe { _mm512_rol_epi32(x, 25) } } +/// # Safety +/// +/// AVX-512F must be available and `r` must be in `0..7`. #[inline(always)] unsafe fn round(v: &mut [__m512i; 16], m: &[__m512i; 16], r: usize) { // SAFETY: Caller guarantees this AVX-512 backend is active; all vector lanes are local registers. @@ -200,6 +219,9 @@ unsafe fn round(v: &mut [__m512i; 16], m: &[__m512i; 16], r: usize) { } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn counter_vec(counter: u64, increment_counter: bool) -> (__m512i, __m512i) { let mask = if increment_counter { !0u64 } else { 0u64 }; @@ -245,11 +267,15 @@ unsafe fn counter_vec(counter: u64, increment_counter: bool) -> (__m512i, __m512 } } +/// # Safety +/// +/// AVX2 must be available and every input must be readable for a complete +/// block starting at `block_offset`. #[inline(always)] unsafe fn transpose_msg_vecs8(inputs: &[*const u8; 8], block_offset: usize) -> [__m256i; 16] { // SAFETY: Caller guarantees each input points to at least one full block at `block_offset`. unsafe { - let stride = 4 * 8; + let stride = 4usize.strict_mul(8); let mut half0 = [ loadu256(inputs[0].add(block_offset)), loadu256(inputs[1].add(block_offset)), @@ -261,18 +287,21 @@ unsafe fn transpose_msg_vecs8(inputs: &[*const u8; 8], block_offset: usize) -> [ loadu256(inputs[7].add(block_offset)), ]; let mut half1 = [ - loadu256(inputs[0].add(block_offset + stride)), - loadu256(inputs[1].add(block_offset + stride)), - loadu256(inputs[2].add(block_offset + stride)), - loadu256(inputs[3].add(block_offset + stride)), - loadu256(inputs[4].add(block_offset + stride)), - loadu256(inputs[5].add(block_offset + stride)), - loadu256(inputs[6].add(block_offset + stride)), - loadu256(inputs[7].add(block_offset + stride)), + loadu256(inputs[0].add(block_offset.strict_add(stride))), + loadu256(inputs[1].add(block_offset.strict_add(stride))), + loadu256(inputs[2].add(block_offset.strict_add(stride))), + loadu256(inputs[3].add(block_offset.strict_add(stride))), + loadu256(inputs[4].add(block_offset.strict_add(stride))), + loadu256(inputs[5].add(block_offset.strict_add(stride))), + loadu256(inputs[6].add(block_offset.strict_add(stride))), + loadu256(inputs[7].add(block_offset.strict_add(stride))), ]; for &input in inputs.iter() { - _mm_prefetch(input.wrapping_add(block_offset + 256).cast::(), _MM_HINT_T0); + _mm_prefetch( + input.wrapping_add(block_offset.strict_add(256)).cast::(), + _MM_HINT_T0, + ); } transpose8x8(&mut half0); @@ -285,6 +314,10 @@ unsafe fn transpose_msg_vecs8(inputs: &[*const u8; 8], block_offset: usize) -> [ } } +/// # Safety +/// +/// AVX-512F, AVX-512DQ, and AVX2 must be available. Every input must be +/// readable for a complete block starting at `block_offset`. #[inline(always)] unsafe fn transpose_msg_vecs16(inputs: &[*const u8; 16], block_offset: usize) -> [__m512i; 16] { // SAFETY: Caller guarantees each input points to at least one full block at `block_offset`. @@ -310,63 +343,6 @@ unsafe fn transpose_msg_vecs16(inputs: &[*const u8; 16], block_offset: usize) -> } } -/// Hash 16 contiguous independent inputs in parallel. -/// -/// This is optimized for the contiguous chunk hashing hot path, where inputs -/// are arranged as `CHUNK_LEN`-byte blocks back-to-back. -/// -/// # Safety -/// Caller must ensure AVX-512 is available, and `input`/`out` are valid for -/// `DEGREE * CHUNK_LEN` and `DEGREE * OUT_LEN` bytes respectively. -#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] -// Match upstream: AVX-512 detection is based on `avx512f` + `avx512vl`. -// The Linux backend delegates to upstream-grade asm, so we intentionally do -// not require BW/DQ here. -#[target_feature(enable = "avx512f,avx512vl,avx2")] -pub(crate) unsafe fn hash16_contiguous(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { - // Delegate to the upstream-grade AVX-512 asm implementation on Linux. - // - // The upstream function accepts an array of input pointers, so we build it - // from the contiguous layout. - debug_assert!(flags <= u8::MAX as u32); - // SAFETY: Caller guarantees AVX-512F/VL+AVX2 availability, `input` readable for 16 chunks, and - // `out` writable for `DEGREE * OUT_LEN`. The asm entrypoint shares that contract. - unsafe { - let inputs = [ - input, - input.add(CHUNK_LEN), - input.add(2 * CHUNK_LEN), - input.add(3 * CHUNK_LEN), - input.add(4 * CHUNK_LEN), - input.add(5 * CHUNK_LEN), - input.add(6 * CHUNK_LEN), - input.add(7 * CHUNK_LEN), - input.add(8 * CHUNK_LEN), - input.add(9 * CHUNK_LEN), - input.add(10 * CHUNK_LEN), - input.add(11 * CHUNK_LEN), - input.add(12 * CHUNK_LEN), - input.add(13 * CHUNK_LEN), - input.add(14 * CHUNK_LEN), - input.add(15 * CHUNK_LEN), - ]; - let flags_start = (flags | super::super::CHUNK_START) as u8; - let flags_end = (flags | super::super::CHUNK_END) as u8; - super::asm::rscrypto_blake3_hash_many_avx512( - inputs.as_ptr(), - DEGREE, - CHUNK_LEN / BLOCK_LEN, - key.as_ptr(), - counter, - true, - flags as u8, - flags_start, - flags_end, - out, - ); - } -} - /// Owned Rust-intrinsic implementation of the 16-way contiguous hash-many kernel. /// /// This stays callable on platforms where production dispatch still prefers @@ -379,15 +355,17 @@ pub(crate) unsafe fn hash16_contiguous(input: *const u8, key: &[u32; 8], counter /// `out` is valid for `DEGREE * OUT_LEN` writable bytes. #[target_feature(enable = "avx512f,avx512vl,avx512dq,avx2")] pub(crate) unsafe fn hash16_owned( - inputs: &[*const u8; DEGREE], - blocks: usize, - key: &[u32; 8], - counter: u64, - increment_counter: bool, - flags: u32, - flags_start: u32, - flags_end: u32, - out: *mut u8, + HashManyRequest { + inputs, + blocks, + key, + counter, + increment_counter, + flags, + flags_start, + flags_end, + out, + }: HashManyRequest<'_, DEGREE>, ) { // SAFETY: 16-way AVX-512 BLAKE3 contiguous hash-many because: // 1. The caller guarantees AVX-512F/VL/DQ plus AVX2 availability. @@ -395,7 +373,7 @@ pub(crate) unsafe fn hash16_owned( // 3. `out` is writable for `DEGREE * OUT_LEN` bytes. // 4. All lane pointers and stores are bounded by fixed-size local arrays. unsafe { - let block_len_vec = set1(BLOCK_LEN as u32); + let block_len_vec = set1(BLOCK_LEN_U32); let iv0 = set1(IV[0]); let iv1 = set1(IV[1]); let iv2 = set1(IV[2]); @@ -419,13 +397,13 @@ pub(crate) unsafe fn hash16_owned( if block == 0 { block_flags |= flags_start; } - if block + 1 == blocks { + if block.strict_add(1) == blocks { block_flags |= flags_end; } let block_flags_vec = set1(block_flags); - let m = transpose_msg_vecs16(inputs, block * BLOCK_LEN); + let m = transpose_msg_vecs16(inputs, block.strict_mul(BLOCK_LEN)); let mut v = [ h_vecs[0], @@ -476,8 +454,8 @@ pub(crate) unsafe fn hash16_owned( transpose8x8(&mut hi); for chunk in 0..8 { - storeu256(lo[chunk], out.add(chunk * OUT_LEN)); - storeu256(hi[chunk], out.add((chunk + 8) * OUT_LEN)); + storeu256(lo[chunk], out.add(chunk.strict_mul(OUT_LEN))); + storeu256(hi[chunk], out.add(chunk.strict_add(8).strict_mul(OUT_LEN))); } } } @@ -518,17 +496,17 @@ pub(crate) unsafe fn hash16_contiguous_owned(input: *const u8, key: &[u32; 8], c input.add(14 * CHUNK_LEN), input.add(15 * CHUNK_LEN), ]; - hash16_owned( - &inputs, - CHUNK_LEN / BLOCK_LEN, + hash16_owned(HashManyRequest { + inputs: &inputs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - super::super::CHUNK_START, - super::super::CHUNK_END, + flags_start: super::super::CHUNK_START, + flags_end: super::super::CHUNK_END, out, - ); + }); } } @@ -569,21 +547,21 @@ pub(crate) unsafe fn root_output_blocks16( ) { #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] { - debug_assert!(flags <= u8::MAX as u32); - debug_assert!(block_len <= u8::MAX as u32); + let block_len = u8::try_from(block_len).expect("BLAKE3 block length must fit the assembly ABI"); + let flags = u8::try_from(flags).expect("BLAKE3 flags must fit the assembly ABI"); // SAFETY: AVX-512 XOF assembly call because: // 1. This target-feature function requires the AVX-512 features used by the wrapper. // 2. `chaining_value` and `block_words` are fixed-size readable arrays. // 3. The caller guarantees `out` is writable for `16 * 64` bytes. - // 4. `block_len` and `flags` are debug-checked to fit the assembly ABI. + // 4. `block_len` and `flags` were converted without loss to the assembly ABI types. // 5. Counters, block length, flags, and output block count are public values. unsafe { super::asm::xof_many_avx512( chaining_value.as_ptr(), block_words.as_ptr().cast(), - block_len as u8, + block_len, counter, - flags as u8, + flags, out, 16, ); @@ -591,6 +569,9 @@ pub(crate) unsafe fn root_output_blocks16( } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] + // SAFETY: Running the intrinsic AVX-512 root-output implementation because this function's + // target-feature contract provides AVX-512F/VL and AVX2, and the caller provides 16 writable + // output blocks. unsafe { let cv_vecs = [ set1(chaining_value[0]), @@ -685,8 +666,8 @@ pub(crate) unsafe fn root_output_blocks16( for i in 0..8 { lo0[i] = _mm512_castsi512_si256(out_words[i]); hi0[i] = _mm512_extracti64x4_epi64(out_words[i], 1); - lo1[i] = _mm512_castsi512_si256(out_words[i + 8]); - hi1[i] = _mm512_extracti64x4_epi64(out_words[i + 8], 1); + lo1[i] = _mm512_castsi512_si256(out_words[i.strict_add(8)]); + hi1[i] = _mm512_extracti64x4_epi64(out_words[i.strict_add(8)], 1); } transpose8x8(&mut lo0); @@ -694,13 +675,13 @@ pub(crate) unsafe fn root_output_blocks16( transpose8x8(&mut lo1); transpose8x8(&mut hi1); - for lane in 0..8 { - let base = out.add(lane * 64); + for lane in 0usize..8 { + let base = out.add(lane.strict_mul(64)); storeu256(lo0[lane], base); storeu256(lo1[lane], base.add(32)); } - for lane in 0..8 { - let base = out.add((lane + 8) * 64); + for lane in 0usize..8 { + let base = out.add(lane.strict_add(8).strict_mul(64)); storeu256(hi0[lane], base); storeu256(hi1[lane], base.add(32)); } @@ -723,21 +704,21 @@ pub(crate) unsafe fn root_output_blocks( blocks: usize, ) { debug_assert!(blocks != 0); - debug_assert!(flags <= u8::MAX as u32); - debug_assert!(block_len <= u8::MAX as u32); + let block_len = u8::try_from(block_len).expect("BLAKE3 block length must fit the assembly ABI"); + let flags = u8::try_from(flags).expect("BLAKE3 flags must fit the assembly ABI"); // SAFETY: AVX-512 XOF assembly call because: // 1. Dispatch only selects this function for the AVX-512 kernel. // 2. The caller guarantees `out` is writable for `blocks * 64` bytes. // 3. `block_words` is a readable 64-byte block and `chaining_value` has 8 words. - // 4. `block_len` and `flags` are debug-checked to fit the assembly ABI. + // 4. `block_len` and `flags` were converted without loss to the assembly ABI types. // 5. Counters, block length, flags, and output block count are public values. unsafe { super::asm::xof_many_avx512( chaining_value.as_ptr(), block_words.as_ptr().cast(), - block_len as u8, + block_len, counter, - flags as u8, + flags, out, blocks, ); @@ -806,7 +787,7 @@ pub(crate) unsafe fn compress_block( let m2 = _mm_loadu_si128(block_words.as_ptr().add(8).cast()); let m3 = _mm_loadu_si128(block_words.as_ptr().add(12).cast()); let [mut row0, mut row1, mut row2, mut row3] = - super::compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + super::compress_pre_sse41_impl(chaining_value, [m0, m1, m2, m3], counter, block_len, flags); let cv_lo = _mm_loadu_si128(chaining_value.as_ptr().cast()); let cv_hi = _mm_loadu_si128(chaining_value.as_ptr().add(4).cast()); @@ -824,33 +805,3 @@ pub(crate) unsafe fn compress_block( out } } - -/// Compress one BLAKE3 block and return only the chaining value (8 words). -/// -/// # Safety -/// Caller must ensure AVX-512F + AVX-512VL + AVX2 + SSE4.1 + SSSE3 are available. -#[target_feature(enable = "avx512f,avx512vl,avx2,sse4.1,ssse3")] -pub(crate) unsafe fn compress_cv_block( - chaining_value: &[u32; 8], - block_words: &[u32; 16], - counter: u64, - block_len: u32, - flags: u32, -) -> [u32; 8] { - // SAFETY: AVX-512/AVX2/SSE4.1/SSSE3 intrinsics are available via this function's - // #[target_feature] attribute. Pointer accesses are to valid fixed-size array references. - unsafe { - let m0 = _mm_loadu_si128(block_words.as_ptr().cast()); - let m1 = _mm_loadu_si128(block_words.as_ptr().add(4).cast()); - let m2 = _mm_loadu_si128(block_words.as_ptr().add(8).cast()); - let m3 = _mm_loadu_si128(block_words.as_ptr().add(12).cast()); - let [row0, row1, row2, row3] = - super::compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); - let row0 = _mm_xor_si128(row0, row2); - let row1 = _mm_xor_si128(row1, row3); - let mut out = [0u32; 8]; - _mm_storeu_si128(out.as_mut_ptr().cast(), row0); - _mm_storeu_si128(out.as_mut_ptr().add(4).cast(), row1); - out - } -} diff --git a/src/hashes/crypto/blake3/x86_64/sse41.rs b/src/hashes/crypto/blake3/x86_64/sse41.rs index d3f36d33..d680af40 100644 --- a/src/hashes/crypto/blake3/x86_64/sse41.rs +++ b/src/hashes/crypto/blake3/x86_64/sse41.rs @@ -1,67 +1,89 @@ //! BLAKE3 x86_64 SSE4.1 throughput kernel (4-way). -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::too_many_arguments)] -#![allow(clippy::many_single_char_names)] - use core::arch::x86_64::*; use super::{ - super::{BLOCK_LEN, IV, MSG_SCHEDULE}, - counter_high, counter_low, + super::{BLOCK_LEN, BLOCK_LEN_U32, IV, MSG_SCHEDULE}, + HashManyRequest, counter_high, counter_low, }; -pub const DEGREE: usize = 4; +pub(crate) const DEGREE: usize = 4; +/// # Safety +/// +/// SSE2 must be available and `src` must be readable for 16 bytes. #[inline(always)] unsafe fn loadu(src: *const u8) -> __m128i { // SAFETY: Caller guarantees `src` is valid to read 16 bytes and has enabled this SSE backend. unsafe { _mm_loadu_si128(src.cast()) } } +/// # Safety +/// +/// SSE2 must be available and `dest` must be writable for 16 bytes. #[inline(always)] unsafe fn storeu(src: __m128i, dest: *mut u8) { // SAFETY: Caller guarantees `dest` is valid to write 16 bytes and has enabled this SSE backend. unsafe { _mm_storeu_si128(dest.cast(), src) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn add(a: __m128i, b: __m128i) -> __m128i { // SAFETY: Caller guarantees the required SSE4.1/SSSE3 feature set for this backend. unsafe { _mm_add_epi32(a, b) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn xor(a: __m128i, b: __m128i) -> __m128i { // SAFETY: Caller guarantees the required SSE4.1/SSSE3 feature set for this backend. unsafe { _mm_xor_si128(a, b) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn set1(x: u32) -> __m128i { // SAFETY: Caller guarantees the required SSE4.1/SSSE3 feature set for this backend. unsafe { _mm_set1_epi32(x.cast_signed()) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn set4(a: u32, b: u32, c: u32, d: u32) -> __m128i { // SAFETY: Caller guarantees the required SSE4.1/SSSE3 feature set for this backend. unsafe { _mm_setr_epi32(a.cast_signed(), b.cast_signed(), c.cast_signed(), d.cast_signed()) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn rot12(a: __m128i) -> __m128i { // SAFETY: Caller guarantees the required SSE4.1/SSSE3 feature set for this backend. unsafe { _mm_or_si128(_mm_srli_epi32(a, 12), _mm_slli_epi32(a, 20)) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn rot7(a: __m128i) -> __m128i { // SAFETY: Caller guarantees the required SSE4.1/SSSE3 feature set for this backend. unsafe { _mm_or_si128(_mm_srli_epi32(a, 7), _mm_slli_epi32(a, 25)) } } +/// # Safety +/// +/// SSE4.1 and SSSE3 must be available, and `r` must be in `0..7`. #[inline(always)] unsafe fn round(v: &mut [__m128i; 16], m: &[__m128i; 16], r: usize, rot16_mask: __m128i, rot8_mask: __m128i) { // SAFETY: Caller guarantees this SSE4.1/SSSE3 backend is active; all vector lanes are local @@ -183,6 +205,9 @@ unsafe fn round(v: &mut [__m128i; 16], m: &[__m128i; 16], r: usize, rot16_mask: } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn transpose_vecs(vecs: &mut [__m128i; DEGREE]) { // SAFETY: Caller guarantees this SSE4.1/SSSE3 backend is active; `vecs` is a valid fixed-size @@ -205,11 +230,15 @@ unsafe fn transpose_vecs(vecs: &mut [__m128i; DEGREE]) { } } +/// # Safety +/// +/// SSE4.1 and SSSE3 must be available. Every input must be readable for a +/// complete block starting at `block_offset`. #[inline(always)] unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) -> [__m128i; 16] { // SAFETY: Caller guarantees each input points to at least one full block at `block_offset`. unsafe { - let stride = 4 * DEGREE; + let stride = 4usize.strict_mul(DEGREE); let mut quarter0 = [ loadu(inputs[0].add(block_offset)), loadu(inputs[1].add(block_offset)), @@ -217,26 +246,29 @@ unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) loadu(inputs[3].add(block_offset)), ]; let mut quarter1 = [ - loadu(inputs[0].add(block_offset + stride)), - loadu(inputs[1].add(block_offset + stride)), - loadu(inputs[2].add(block_offset + stride)), - loadu(inputs[3].add(block_offset + stride)), + loadu(inputs[0].add(block_offset.strict_add(stride))), + loadu(inputs[1].add(block_offset.strict_add(stride))), + loadu(inputs[2].add(block_offset.strict_add(stride))), + loadu(inputs[3].add(block_offset.strict_add(stride))), ]; let mut quarter2 = [ - loadu(inputs[0].add(block_offset + 2 * stride)), - loadu(inputs[1].add(block_offset + 2 * stride)), - loadu(inputs[2].add(block_offset + 2 * stride)), - loadu(inputs[3].add(block_offset + 2 * stride)), + loadu(inputs[0].add(block_offset.strict_add(2usize.strict_mul(stride)))), + loadu(inputs[1].add(block_offset.strict_add(2usize.strict_mul(stride)))), + loadu(inputs[2].add(block_offset.strict_add(2usize.strict_mul(stride)))), + loadu(inputs[3].add(block_offset.strict_add(2usize.strict_mul(stride)))), ]; let mut quarter3 = [ - loadu(inputs[0].add(block_offset + 3 * stride)), - loadu(inputs[1].add(block_offset + 3 * stride)), - loadu(inputs[2].add(block_offset + 3 * stride)), - loadu(inputs[3].add(block_offset + 3 * stride)), + loadu(inputs[0].add(block_offset.strict_add(3usize.strict_mul(stride)))), + loadu(inputs[1].add(block_offset.strict_add(3usize.strict_mul(stride)))), + loadu(inputs[2].add(block_offset.strict_add(3usize.strict_mul(stride)))), + loadu(inputs[3].add(block_offset.strict_add(3usize.strict_mul(stride)))), ]; for &input in inputs.iter() { - _mm_prefetch(input.wrapping_add(block_offset + 256).cast::(), _MM_HINT_T0); + _mm_prefetch( + input.wrapping_add(block_offset.strict_add(256)).cast::(), + _MM_HINT_T0, + ); } transpose_vecs(&mut quarter0); @@ -265,6 +297,9 @@ unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn load_counters(counter: u64, increment_counter: bool) -> (__m128i, __m128i) { let mask = if increment_counter { !0u64 } else { 0u64 }; @@ -294,15 +329,17 @@ unsafe fn load_counters(counter: u64, increment_counter: bool) -> (__m128i, __m1 /// for `blocks * BLOCK_LEN` bytes. #[target_feature(enable = "sse4.1,ssse3")] pub(crate) unsafe fn hash4( - inputs: &[*const u8; DEGREE], - blocks: usize, - key: &[u32; 8], - counter: u64, - increment_counter: bool, - flags: u32, - flags_start: u32, - flags_end: u32, - out: *mut u8, + HashManyRequest { + inputs, + blocks, + key, + counter, + increment_counter, + flags, + flags_start, + flags_end, + out, + }: HashManyRequest<'_, DEGREE>, ) { // SAFETY: Caller guarantees SSE4.1/SSSE3 availability, valid input pointers for `blocks * // BLOCK_LEN`, and `out` writable for `DEGREE * OUT_LEN`. @@ -310,7 +347,7 @@ pub(crate) unsafe fn hash4( let rot16_mask = _mm_setr_epi8(2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13); let rot8_mask = _mm_setr_epi8(1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12); - let block_len_vec = set1(BLOCK_LEN as u32); + let block_len_vec = set1(BLOCK_LEN_U32); let iv0 = set1(IV[0]); let iv1 = set1(IV[1]); let iv2 = set1(IV[2]); @@ -334,12 +371,12 @@ pub(crate) unsafe fn hash4( if block == 0 { block_flags |= flags_start; } - if block + 1 == blocks { + if block.strict_add(1) == blocks { block_flags |= flags_end; } let block_flags_vec = set1(block_flags); - let msg_vecs = transpose_msg_vecs(inputs, block * BLOCK_LEN); + let msg_vecs = transpose_msg_vecs(inputs, block.strict_mul(BLOCK_LEN)); let mut v = [ h_vecs[0], @@ -515,7 +552,7 @@ pub(crate) unsafe fn root_output_blocks4( transpose_vecs(&mut g3); for lane in 0..DEGREE { - let base = out.add(lane * 64); + let base = out.add(lane.strict_mul(64)); storeu(g0[lane], base); storeu(g1[lane], base.add(16)); storeu(g2[lane], base.add(32)); @@ -560,8 +597,8 @@ unsafe fn root_output_blocks1_from_ptr( let mut row3 = _mm_set_epi32( flags.cast_signed(), block_len.cast_signed(), - (counter >> 32) as i32, - counter as i32, + counter_high(counter).cast_signed(), + counter_low(counter).cast_signed(), ); // Load message words row-wise diff --git a/src/hashes/crypto/cshake.rs b/src/hashes/crypto/cshake.rs index cbbb0ac8..c2d293c5 100644 --- a/src/hashes/crypto/cshake.rs +++ b/src/hashes/crypto/cshake.rs @@ -1,7 +1,5 @@ //! cSHAKE128 and cSHAKE256 (SP 800-185). -#![allow(clippy::indexing_slicing)] // Fixed-width prefix encodings and rate-sized zero padding. - use super::{ keccak::{KeccakCore, KeccakXof}, sp800185::{RATE_128, RATE_256, absorb_bytepad, encoded_string_len, left_encode}, @@ -151,7 +149,7 @@ mod tests { let mut hex = String::new(); for byte in out { - write!(&mut hex, "{byte:02x}").unwrap(); + write!(&mut hex, "{byte:02x}").expect("writing hexadecimal to String must succeed"); } hex } diff --git a/src/hashes/crypto/dispatch_util.rs b/src/hashes/crypto/dispatch_util.rs index a14c49ba..b95f7e30 100644 --- a/src/hashes/crypto/dispatch_util.rs +++ b/src/hashes/crypto/dispatch_util.rs @@ -35,20 +35,12 @@ impl SizeClassDispatch { #[must_use] #[cfg(feature = "sha2")] pub(crate) fn len_hint_from_u64(v: u64) -> usize { - if (v as usize) as u64 == v { - v as usize - } else { - usize::MAX - } + usize::try_from(v).unwrap_or(usize::MAX) } #[inline] #[must_use] #[cfg(feature = "sha2")] pub(crate) fn len_hint_from_u128(v: u128) -> usize { - if (v as usize) as u128 == v { - v as usize - } else { - usize::MAX - } + usize::try_from(v).unwrap_or(usize::MAX) } diff --git a/src/hashes/crypto/keccak.rs b/src/hashes/crypto/keccak.rs index ec54dc6e..74dc4ea6 100644 --- a/src/hashes/crypto/keccak.rs +++ b/src/hashes/crypto/keccak.rs @@ -3,14 +3,12 @@ //! This module intentionally exposes only the minimum surface needed by SHA-3, //! SHAKE, and SP800-185 derived constructions. -#![allow(clippy::indexing_slicing)] // Keccak state is fixed-size; indexing is audited - #[cfg(all(target_arch = "aarch64", not(miri)))] pub(crate) mod aarch64; -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] #[doc(hidden)] pub(crate) mod dispatch; -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] #[doc(hidden)] pub(crate) mod dispatch_tables; #[cfg(test)] @@ -73,16 +71,17 @@ const RC: [u64; KECCAKF_ROUNDS] = [ /// all rounds. #[cfg(not(target_arch = "aarch64"))] #[inline] -#[allow(unused_assignments)] // final ρ+π iteration assigns `last` which is intentionally unused pub(crate) fn keccakf_portable(state: &mut [u64; 25]) { // ρ+π chain: swap state[PI] with rotated last, hardcoded for constant folding. macro_rules! rho_pi { - ($state:ident, $last:ident, $(($pi:expr, $rho:expr)),+ $(,)?) => { - $( - let tmp = $state[$pi]; - $state[$pi] = $last.rotate_left($rho); - $last = tmp; - )+ + ($state:ident, $last:ident, ($pi:expr, $rho:expr) $(,)?) => { + $state[$pi] = $last.rotate_left($rho); + }; + ($state:ident, $last:ident, ($pi:expr, $rho:expr), $(($rest_pi:expr, $rest_rho:expr)),+ $(,)?) => { + let tmp = $state[$pi]; + $state[$pi] = $last.rotate_left($rho); + $last = tmp; + rho_pi!($state, $last, $(($rest_pi, $rest_rho)),+); }; } @@ -416,6 +415,16 @@ pub(crate) fn keccakf_portable(state: &mut [u64; 25]) { state[24] = a24; } +#[inline(always)] +fn read_unaligned_block_lane(block: &[u8; RATE], lane: usize) -> u64 { + assert!(lane < RATE / 8, "Keccak lane must be inside the rate block"); + let offset = lane.strict_mul(8); + + // SAFETY: The assertion establishes `lane * 8 + 8 <= RATE`; the pointer retains `block`'s provenance, and + // `read_unaligned` accepts the byte array's alignment. + u64::from_le(unsafe { core::ptr::read_unaligned(block.as_ptr().add(offset).cast::()) }) +} + /// aarch64: fused absorb + Keccak-f[1600] — XOR block data during register load. /// /// Loads `state[i] ^ block_lane_i` directly into named register variables for @@ -427,20 +436,10 @@ pub(crate) fn keccakf_portable(state: &mut [u64; 25]) { /// eliminates all `if lane < lanes` branches — the result is straight-line code. #[cfg(target_arch = "aarch64")] #[inline] -#[allow(dead_code)] // Only used by non-Miri platform permuters with absorb-block overrides. fn keccakf_absorb_portable(state: &mut [u64; 25], block: &[u8; RATE]) { debug_assert_eq!(RATE % 8, 0); let lanes = RATE / 8; - #[inline(always)] - fn read_block_lane(block: *const u8, i: usize) -> u64 { - // SAFETY: caller guarantees `i < block.len() / 8`, so the read is in bounds. - // `read_unaligned` supports the 1-byte alignment of `[u8; RATE]`. - u64::from_le(unsafe { core::ptr::read_unaligned(block.cast::().add(i)) }) - } - - let ptr = block.as_ptr(); - // Fused load: `state[i] ^ block_lane_i` for absorbed (rate) lanes, // plain `state[i]` for capacity lanes. Each `$i` is a literal, so // `$i < lanes` (where `lanes = RATE / 8`, a compile-time constant) @@ -448,7 +447,7 @@ fn keccakf_absorb_portable(state: &mut [u64; 25], block: &[u8 macro_rules! fused_load { ($i:expr) => { if $i < lanes { - state[$i] ^ read_block_lane(ptr, $i) + state[$i] ^ read_unaligned_block_lane(block, $i) } else { state[$i] } @@ -548,8 +547,8 @@ pub(crate) trait Permuter: Copy { /// Permute three independent states in parallel. /// Default: one paired permutation and one single-state permutation. + #[cfg(feature = "ml-kem")] #[inline(always)] - #[allow(dead_code)] fn permute_x3(self, state_a: &mut [u64; 25], state_b: &mut [u64; 25], state_c: &mut [u64; 25], len_hint: usize) { self.permute_x2(state_a, state_b, len_hint); self.permute(state_c, len_hint); @@ -557,8 +556,8 @@ pub(crate) trait Permuter: Copy { /// Permute four independent states in parallel. /// Default: two paired permutations. + #[cfg(feature = "ml-kem")] #[inline(always)] - #[allow(dead_code)] fn permute_x4( self, state_a: &mut [u64; 25], @@ -584,10 +583,17 @@ pub(crate) trait Permuter: Copy { /// Direct-call permuter using the portable scalar kernel. No function pointer /// indirection — LLVM can inline `keccakf_portable` into the absorb loop. +#[cfg(any( + miri, + not(any(target_arch = "aarch64", target_arch = "x86_64", target_arch = "s390x")) +))] #[derive(Clone, Copy, Default)] -#[allow(dead_code)] // Reference/test permuter is target- and feature-combination dependent. pub(crate) struct InlinePermuter; +#[cfg(any( + miri, + not(any(target_arch = "aarch64", target_arch = "x86_64", target_arch = "s390x")) +))] impl Permuter for InlinePermuter { #[inline(always)] fn permute(self, state: &mut [u64; 25], _len_hint: usize) { @@ -636,6 +642,7 @@ impl Permuter for X86Permuter { } } + #[cfg(feature = "ml-kem")] #[inline(always)] fn permute_x4( self, @@ -672,7 +679,7 @@ pub(crate) struct Aarch64Permuter; #[derive(Clone, Copy)] pub(crate) struct Aarch64Permuter { has_sha3: bool, - #[cfg(target_os = "linux")] + #[cfg(all(target_os = "linux", feature = "ml-kem"))] has_sve2_sha3: bool, } @@ -684,7 +691,7 @@ impl Default for Aarch64Permuter { let caps = crate::platform::caps(); Self { has_sha3: caps.has(aarch64_caps::SHA3), - #[cfg(target_os = "linux")] + #[cfg(all(target_os = "linux", feature = "ml-kem"))] has_sve2_sha3: caps.has(aarch64_caps::SVE2_SHA3), } } @@ -720,20 +727,22 @@ impl Permuter for Aarch64Permuter { aarch64::keccakf_aarch64_sha3_x2(state_a, state_b); } + #[cfg(feature = "ml-kem")] #[inline(always)] fn permute_x3(self, state_a: &mut [u64; 25], state_b: &mut [u64; 25], state_c: &mut [u64; 25], _len_hint: usize) { - #[cfg(all(target_os = "linux", not(target_vendor = "apple")))] + #[cfg(all(target_os = "linux", not(target_vendor = "apple"), feature = "ml-kem"))] { aarch64::keccakf_aarch64_sha3_x3_hybrid(state_a, state_b, state_c); } - #[cfg(not(all(target_os = "linux", not(target_vendor = "apple"))))] + #[cfg(not(all(target_os = "linux", not(target_vendor = "apple"), feature = "ml-kem")))] { self.permute_x2(state_a, state_b, _len_hint); self.permute(state_c, _len_hint); } } + #[cfg(feature = "ml-kem")] #[inline(always)] fn permute_x4( self, @@ -743,7 +752,7 @@ impl Permuter for Aarch64Permuter { state_d: &mut [u64; 25], _len_hint: usize, ) { - #[cfg(target_os = "linux")] + #[cfg(all(target_os = "linux", feature = "ml-kem"))] { use crate::platform::caps::aarch64 as aarch64_caps; if crate::platform::caps().has(aarch64_caps::SVE2_SHA3) @@ -753,13 +762,13 @@ impl Permuter for Aarch64Permuter { } } - #[cfg(all(target_os = "linux", not(target_vendor = "apple")))] + #[cfg(all(target_os = "linux", not(target_vendor = "apple"), feature = "ml-kem"))] { aarch64::keccakf_aarch64_sha3_x3_hybrid(state_a, state_b, state_c); keccakf_portable(state_d); } - #[cfg(not(all(target_os = "linux", not(target_vendor = "apple"))))] + #[cfg(not(all(target_os = "linux", not(target_vendor = "apple"), feature = "ml-kem")))] { self.permute_x2(state_a, state_b, _len_hint); self.permute_x2(state_c, state_d, _len_hint); @@ -820,9 +829,10 @@ impl Permuter for Aarch64Permuter { } } + #[cfg(feature = "ml-kem")] #[inline(always)] fn permute_x3(self, state_a: &mut [u64; 25], state_b: &mut [u64; 25], state_c: &mut [u64; 25], len_hint: usize) { - #[cfg(all(target_os = "linux", not(target_vendor = "apple")))] + #[cfg(all(target_os = "linux", not(target_vendor = "apple"), feature = "ml-kem"))] if self.has_sha3 { aarch64::keccakf_aarch64_sha3_x3_hybrid(state_a, state_b, state_c); return; @@ -832,6 +842,7 @@ impl Permuter for Aarch64Permuter { self.permute(state_c, len_hint); } + #[cfg(feature = "ml-kem")] #[inline(always)] fn permute_x4( self, @@ -841,12 +852,12 @@ impl Permuter for Aarch64Permuter { state_d: &mut [u64; 25], len_hint: usize, ) { - #[cfg(target_os = "linux")] + #[cfg(all(target_os = "linux", feature = "ml-kem"))] if self.has_sve2_sha3 && aarch64::keccakf_aarch64_sve2_sha3_x4(state_a, state_b, state_c, state_d) { return; } - #[cfg(all(target_os = "linux", not(target_vendor = "apple")))] + #[cfg(all(target_os = "linux", not(target_vendor = "apple"), feature = "ml-kem"))] if self.has_sha3 { aarch64::keccakf_aarch64_sha3_x3_hybrid(state_a, state_b, state_c); keccakf_portable(state_d); @@ -960,25 +971,6 @@ impl Default for KeccakCoreImpl Default for KeccakCoreImpl { - #[inline] - fn default() -> Self { - Self { - state: [0u64; 25], - buf_len: 0, - permuter: InlinePermuter, - } - } -} - impl Drop for KeccakCoreImpl { fn drop(&mut self) { if ZEROIZE { @@ -997,7 +989,8 @@ impl KeccakCoreImpl(&mut self.state, self.buf_len, data); self.buf_len = self.buf_len.strict_add(data.len()); return; @@ -1005,7 +998,7 @@ impl KeccakCoreImpl(&mut self.state, self.buf_len, &data[..take]); self.buf_len = self.buf_len.strict_add(take); data = &data[take..]; @@ -1076,21 +1069,16 @@ impl KeccakCoreImpl KeccakCoreImpl { - pub(crate) fn into_xof(self, ds: u8) -> KeccakXofImpl { - let mut this = core::mem::ManuallyDrop::new(self); - let inner = &mut *this; - let permuter = inner.permuter; - let buf_len = inner.buf_len; + pub(crate) fn into_xof(mut self, ds: u8) -> KeccakXofImpl { + let permuter = self.permuter; + let buf_len = self.buf_len; debug_assert!(buf_len < RATE, "buf_len={} should be < RATE={}", buf_len, RATE); - pad_absorbed_state::(&mut inner.state, buf_len, ds); - permuter.permute(&mut inner.state, 0); + pad_absorbed_state::(&mut self.state, buf_len, ds); + permuter.permute(&mut self.state, 0); KeccakXofImpl { - // SAFETY: this public Keccak core is non-zeroizing, and `this` is - // `ManuallyDrop`, so moving the finalized state into the reader does not - // bypass any required cleanup. - state: unsafe { core::ptr::read(&inner.state) }, + state: self.state, pos: 0, permuter, } @@ -1147,14 +1135,10 @@ pub(crate) fn oneshot_fixed(ds: u8, data: & fn xor_block_into(state: &mut [u64; 25], block: &[u8; RATE]) { debug_assert_eq!(RATE % 8, 0); let lanes = RATE / 8; - let ptr = block.as_ptr() as *const u64; let mut i = 0usize; while i < lanes { - // SAFETY: `RATE % 8 == 0` and `i < lanes == RATE / 8`, so this reads within `block`; - // `read_unaligned` supports the 1-byte alignment of `[u8; RATE]`. - let v = unsafe { core::ptr::read_unaligned(ptr.add(i)) }; - state[i] ^= u64::from_le(v); - i += 1; + state[i] ^= read_unaligned_block_lane(block, i); + i = i.strict_add(1); } } @@ -1163,7 +1147,7 @@ fn xor_block_into(state: &mut [u64; 25], block: &[u8; RATE]) fn xor_bytes_into_state(state: &mut [u64; 25], mut offset: usize, mut data: &[u8]) { debug_assert_eq!(RATE % 8, 0); debug_assert!(offset <= RATE); - debug_assert!(data.len() <= RATE - offset); + debug_assert!(data.len() <= RATE.strict_sub(offset)); while !data.is_empty() && (offset & 7) != 0 { state[offset / 8] ^= (data[0] as u64) << ((offset & 7).strict_mul(8)); @@ -1195,7 +1179,7 @@ fn pad_absorbed_state(state: &mut [u64; 25], pos: usize, ds: state[pos / 8] ^= (ds as u64) << ((pos & 7).strict_mul(8)); - let last = RATE - 1; + let last = RATE.strict_sub(1); state[last / 8] ^= 0x80_u64 << ((last & 7).strict_mul(8)); } @@ -1310,7 +1294,7 @@ fn xof_seeded_32_2_base_state(ds: u8, seed: &[u8; 32]) -> [u6 state[3] = u64::from_le_bytes(seed_words[3]); state[4] = u64::from(ds) << 16; - let last = RATE - 1; + let last = RATE.strict_sub(1); state[last / 8] ^= 0x80_u64 << ((last & 7).strict_mul(8)); state } @@ -1342,7 +1326,7 @@ fn xof_seeded_32_1_state(ds: u8, seed: &[u8; 32], x: u8) -> [ state[3] = u64::from_le_bytes(seed_words[3]); state[4] = u64::from(x) | (u64::from(ds) << 8); - let last = RATE - 1; + let last = RATE.strict_sub(1); state[last / 8] ^= 0x80_u64 << ((last & 7).strict_mul(8)); state } @@ -1680,13 +1664,13 @@ impl KeccakXofImpl KeccakXofImpl KeccakXofImpl KeccakXofImpl KeccakXofImpl KeccakXofImpl(state: &mut [u64; 25], block: &[u8; RATE]) { @@ -431,7 +432,7 @@ pub(crate) fn keccakf_aarch64_sha3_absorb_single(state: &mut #[target_feature(enable = "sha3")] unsafe fn keccakf_sha3_absorb_blocks_impl(state: &mut [u64; 25], blocks: &[u8]) { debug_assert_eq!(RATE % 8, 0); - debug_assert_eq!(blocks.len() % RATE, 0); + debug_assert!(blocks.len().is_multiple_of(RATE)); let lanes = RATE / 8; let z = vcreate_u64(0); @@ -545,7 +546,17 @@ unsafe fn keccakf_sha3_absorb_blocks_impl(state: &mut [u64; 2 #[cfg(target_arch = "aarch64")] #[inline] pub(crate) fn keccakf_aarch64_sha3_absorb_blocks(state: &mut [u64; 25], blocks: &[u8]) { - // SAFETY: Dispatch verifies aarch64::SHA3 capability before calling. + assert!( + RATE != 0 && RATE.is_multiple_of(8), + "Keccak rate must be a positive whole number of lanes" + ); + assert!( + blocks.len().is_multiple_of(RATE), + "Keccak batch must contain complete rate blocks" + ); + + // SAFETY: Dispatch verifies aarch64::SHA3 capability before calling. The assertions above establish the nonzero, + // whole-lane rate and complete-block partition required by the pointer loop. unsafe { keccakf_sha3_absorb_blocks_impl::(state, blocks) } } @@ -553,6 +564,10 @@ pub(crate) fn keccakf_aarch64_sha3_absorb_blocks(state: &mut /// Combine lane 0 from `state_a[i]` and lane 1 from `state_b[i]` into one /// uint64x2_t register. +/// +/// # Safety +/// +/// The caller must ensure the current CPU supports NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn combine_lanes(a: u64, b: u64) -> uint64x2_t { @@ -689,7 +704,7 @@ pub(crate) fn keccakf_aarch64_sha3_x2(state_a: &mut [u64; 25], state_b: &mut [u6 /// # Safety /// /// Caller must ensure `sha3` target feature is available. -#[cfg(all(target_arch = "aarch64", target_os = "linux"))] +#[cfg(all(target_arch = "aarch64", target_os = "linux", feature = "ml-kem"))] #[target_feature(enable = "sha3")] unsafe fn keccakf_sha3_x3_hybrid_impl(state_a: &mut [u64; 25], state_b: &mut [u64; 25], state_c: &mut [u64; 25]) { // SAFETY: NEON + SHA3 CE intrinsics are available because: @@ -840,7 +855,7 @@ unsafe fn keccakf_sha3_x3_hybrid_impl(state_a: &mut [u64; 25], state_b: &mut [u6 } } -#[cfg(all(target_arch = "aarch64", target_os = "linux"))] +#[cfg(all(target_arch = "aarch64", target_os = "linux", feature = "ml-kem"))] #[inline] pub(crate) fn keccakf_aarch64_sha3_x3_hybrid( state_a: &mut [u64; 25], @@ -853,7 +868,7 @@ pub(crate) fn keccakf_aarch64_sha3_x3_hybrid( unsafe { keccakf_sha3_x3_hybrid_impl(state_a, state_b, state_c) } } -#[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri)))] +#[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri), feature = "ml-kem"))] unsafe extern "C" { fn rscrypto_keccakf1600_aarch64_sve2_sha3_x4( state_a: *mut u64, @@ -867,7 +882,7 @@ unsafe extern "C" { /// /// Returns `false` when the runtime SVE vector length is too small for four /// 64-bit lanes. Callers must still gate this on `aarch64::SVE2_SHA3`. -#[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri)))] +#[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri), feature = "ml-kem"))] #[inline] pub(crate) fn keccakf_aarch64_sve2_sha3_x4( state_a: &mut [u64; 25], diff --git a/src/hashes/crypto/keccak/dispatch.rs b/src/hashes/crypto/keccak/dispatch.rs index 41eeb6a6..23526478 100644 --- a/src/hashes/crypto/keccak/dispatch.rs +++ b/src/hashes/crypto/keccak/dispatch.rs @@ -1,5 +1,3 @@ -#![cfg_attr(test, allow(dead_code))] - use super::{ dispatch_tables::DispatchTable, kernels::{Keccakf1600KernelId, permute_fn, required_caps}, @@ -96,7 +94,7 @@ fn select(d: &ActiveDispatch, len: usize) -> (PermuteFn, &'static str) { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] -pub fn kernel_name_for_len(len: usize) -> &'static str { +pub(crate) fn kernel_name_for_len(len: usize) -> &'static str { #[cfg(target_arch = "s390x")] { use crate::platform::caps::s390x; diff --git a/src/hashes/crypto/keccak/dispatch_tables.rs b/src/hashes/crypto/keccak/dispatch_tables.rs index 4047b11e..dfde8615 100644 --- a/src/hashes/crypto/keccak/dispatch_tables.rs +++ b/src/hashes/crypto/keccak/dispatch_tables.rs @@ -1,17 +1,15 @@ -#![cfg_attr(test, allow(dead_code))] - //! Tuned dispatch tables for Keccak-f[1600]. //! //! This table controls which permutation kernel is used by SHA-3/SHAKE and //! SP800-185 derived constructions. -pub use super::kernels::Keccakf1600KernelId as KernelId; +pub(crate) use super::kernels::Keccakf1600KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -19,7 +17,7 @@ pub struct DispatchTable { pub l: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -31,7 +29,7 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { // across both NEON lanes. SHA3 CE is used by the two-state interleaved path, // where each lane carries an independent state. #[cfg(target_arch = "aarch64")] -pub static AARCH64_SHA3_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_SHA3_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -41,7 +39,7 @@ pub static AARCH64_SHA3_TABLE: DispatchTable = DispatchTable { #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { #[cfg(target_arch = "aarch64")] { use crate::platform::caps::aarch64; @@ -50,6 +48,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D } } - #[allow(unreachable_code)] + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/keccak/kernel_test.rs b/src/hashes/crypto/keccak/kernel_test.rs index 1472e019..19d2bb58 100644 --- a/src/hashes/crypto/keccak/kernel_test.rs +++ b/src/hashes/crypto/keccak/kernel_test.rs @@ -3,7 +3,7 @@ use alloc::vec::Vec; use super::kernels::{ALL, permute_fn, required_caps}; #[derive(Clone, Debug)] -pub struct KernelResult { +pub(crate) struct KernelResult { pub name: &'static str, pub state: [u64; 25], } @@ -24,7 +24,7 @@ fn state_from_bytes(data: &[u8]) -> [u64; 25] { } #[must_use] -pub fn run_all_keccakf1600_kernels(data: &[u8]) -> Vec { +pub(crate) fn run_all_keccakf1600_kernels(data: &[u8]) -> Vec { let caps = crate::platform::caps(); let mut out = Vec::with_capacity(ALL.len()); let init = state_from_bytes(data); @@ -44,7 +44,7 @@ pub fn run_all_keccakf1600_kernels(data: &[u8]) -> Vec { out } -pub fn verify_keccakf1600_kernels(data: &[u8]) -> Result<(), &'static str> { +pub(crate) fn verify_keccakf1600_kernels(data: &[u8]) -> Result<(), &'static str> { let results = run_all_keccakf1600_kernels(data); let Some(first) = results.first() else { return Ok(()); @@ -219,7 +219,7 @@ mod tests { /// Verify the production aarch64 x3 dispatch matches three independent portable runs. #[test] - #[cfg(all(target_arch = "aarch64", not(miri)))] + #[cfg(all(feature = "ml-kem", target_arch = "aarch64", not(miri)))] fn keccakf1600_aarch64_platform_x3_matches_portable() { use super::super::Permuter; @@ -260,7 +260,7 @@ mod tests { /// Verify Linux-targeted hybrid x3 kernel matches three independent portable runs. #[test] - #[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri)))] + #[cfg(all(target_arch = "aarch64", target_os = "linux", feature = "ml-kem", not(miri)))] fn keccakf1600_sha3_x3_hybrid_matches_portable() { let caps = crate::platform::caps(); if !caps.has(crate::platform::caps::aarch64::SHA3) { @@ -287,7 +287,7 @@ mod tests { /// Verify SVE2-SHA3 4-state kernel matches four independent portable runs. #[test] - #[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri)))] + #[cfg(all(target_arch = "aarch64", target_os = "linux", feature = "ml-kem", not(miri)))] fn keccakf1600_sve2_sha3_x4_matches_portable() { let caps = crate::platform::caps(); if !caps.has(crate::platform::caps::aarch64::SVE2_SHA3) { @@ -331,7 +331,7 @@ mod tests { let len = RATE.strict_mul(4); let mut blocks = Vec::with_capacity(len); for i in 0..len { - blocks.push(((i.strict_mul(37).strict_add(11)) & 0xff) as u8); + blocks.push(i.strict_mul(37).strict_add(11).to_le_bytes()[0]); } let mut expected = state_from_bytes(b"batch-absorb-reference-state"); @@ -354,4 +354,21 @@ mod tests { assert_rate::<144>(); assert_rate::<168>(); } + + #[test] + #[cfg(all(target_arch = "aarch64", not(miri)))] + #[should_panic(expected = "Keccak rate must be a positive whole number of lanes")] + fn keccakf1600_absorb_blocks_rejects_zero_rate() { + let mut state = [0u64; 25]; + super::super::aarch64::keccakf_aarch64_sha3_absorb_blocks::<0>(&mut state, &[]); + } + + #[test] + #[cfg(all(target_arch = "aarch64", not(miri)))] + #[should_panic(expected = "Keccak batch must contain complete rate blocks")] + fn keccakf1600_absorb_blocks_rejects_partial_block() { + let mut state = [0u64; 25]; + let blocks = [0u8; 73]; + super::super::aarch64::keccakf_aarch64_sha3_absorb_blocks::<72>(&mut state, &blocks); + } } diff --git a/src/hashes/crypto/keccak/kernels.rs b/src/hashes/crypto/keccak/kernels.rs index a842ee7a..2a3b1dc8 100644 --- a/src/hashes/crypto/keccak/kernels.rs +++ b/src/hashes/crypto/keccak/kernels.rs @@ -4,14 +4,14 @@ use crate::platform::Caps; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Keccakf1600KernelId { +pub(crate) enum Keccakf1600KernelId { Portable = 0, } impl Keccakf1600KernelId { #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", } @@ -19,7 +19,7 @@ impl Keccakf1600KernelId { } #[must_use] -pub fn permute_fn(id: Keccakf1600KernelId) -> fn(&mut [u64; 25]) { +pub(crate) fn permute_fn(id: Keccakf1600KernelId) -> fn(&mut [u64; 25]) { match id { Keccakf1600KernelId::Portable => keccakf_portable, } @@ -27,11 +27,11 @@ pub fn permute_fn(id: Keccakf1600KernelId) -> fn(&mut [u64; 25]) { /// All kernel IDs for agreement testing. #[cfg(test)] -pub const ALL: &[Keccakf1600KernelId] = &[Keccakf1600KernelId::Portable]; +pub(crate) const ALL: &[Keccakf1600KernelId] = &[Keccakf1600KernelId::Portable]; #[inline] #[must_use] -pub const fn required_caps(id: Keccakf1600KernelId) -> Caps { +pub(crate) const fn required_caps(id: Keccakf1600KernelId) -> Caps { match id { Keccakf1600KernelId::Portable => Caps::NONE, } diff --git a/src/hashes/crypto/keccak/s390x.rs b/src/hashes/crypto/keccak/s390x.rs index 3611d22d..2854c527 100644 --- a/src/hashes/crypto/keccak/s390x.rs +++ b/src/hashes/crypto/keccak/s390x.rs @@ -25,8 +25,6 @@ //! //! Requires MSA8 facility (CPACF, z14+). Caller must verify `s390x::MSA8`. -#![allow(unsafe_code)] - /// Map Keccak rate (bytes) to KIMD function code. /// /// For rate=136 (shared by SHA3-256 and SHAKE-256), returns SHA3-256's code (33). diff --git a/src/hashes/crypto/keccak/x86_64.rs b/src/hashes/crypto/keccak/x86_64.rs index 2d6fcbe4..54873b0a 100644 --- a/src/hashes/crypto/keccak/x86_64.rs +++ b/src/hashes/crypto/keccak/x86_64.rs @@ -11,6 +11,16 @@ #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; +#[inline(always)] +const fn u64_to_i64_bits(value: u64) -> i64 { + i64::from_ne_bytes(value.to_ne_bytes()) +} + +#[inline(always)] +const fn i64_to_u64_bits(value: i64) -> u64 { + u64::from_ne_bytes(value.to_ne_bytes()) +} + #[cfg(target_arch = "x86_64")] macro_rules! rol { ($value:expr, $left:literal) => {{ _mm_rol_epi64::<$left>($value) }}; @@ -21,12 +31,12 @@ macro_rules! chi { ($a:expr, $b:expr, $c:expr) => {{ _mm_ternarylogic_epi64($a, $b, $c, 0xD2) }}; } -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", feature = "ml-kem"))] macro_rules! rol4 { ($value:expr, $left:literal) => {{ _mm256_rol_epi64::<$left>($value) }}; } -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", feature = "ml-kem"))] macro_rules! chi4 { ($a:expr, $b:expr, $c:expr) => {{ _mm256_ternarylogic_epi64($a, $b, $c, 0xD2) }}; } @@ -42,14 +52,14 @@ macro_rules! chi4 { unsafe fn keccakf_x86_avx512_x2_impl(state_a: &mut [u64; 25], state_b: &mut [u64; 25]) { macro_rules! load { ($i:literal) => { - _mm_set_epi64x(state_b[$i] as i64, state_a[$i] as i64) + _mm_set_epi64x(u64_to_i64_bits(state_b[$i]), u64_to_i64_bits(state_a[$i])) }; } macro_rules! store { ($i:literal, $value:expr) => {{ - state_a[$i] = _mm_extract_epi64::<0>($value) as u64; - state_b[$i] = _mm_extract_epi64::<1>($value) as u64; + state_a[$i] = i64_to_u64_bits(_mm_extract_epi64::<0>($value)); + state_b[$i] = i64_to_u64_bits(_mm_extract_epi64::<1>($value)); }}; } @@ -186,7 +196,7 @@ unsafe fn keccakf_x86_avx512_x2_impl(state_a: &mut [u64; 25], state_b: &mut [u64 a23 = chi!(b23, b24, b20); a24 = chi!(b24, b20, b21); - a0 = xor!(a0, _mm_set1_epi64x(rc as i64)); + a0 = xor!(a0, _mm_set1_epi64x(u64_to_i64_bits(rc))); } store!(0, a0); @@ -235,7 +245,7 @@ pub(crate) unsafe fn keccakf_x86_avx512_x2(state_a: &mut [u64; 25], state_b: &mu /// # Safety /// /// Caller must ensure `avx512f`, `avx512vl`, and `sse4.1` CPU features are available. -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", feature = "ml-kem"))] #[target_feature(enable = "avx512f,avx512vl,sse4.1")] #[inline] unsafe fn keccakf_x86_avx512_x4_impl( @@ -247,20 +257,20 @@ unsafe fn keccakf_x86_avx512_x4_impl( macro_rules! load { ($i:literal) => { _mm256_set_epi64x( - state_d[$i] as i64, - state_c[$i] as i64, - state_b[$i] as i64, - state_a[$i] as i64, + u64_to_i64_bits(state_d[$i]), + u64_to_i64_bits(state_c[$i]), + u64_to_i64_bits(state_b[$i]), + u64_to_i64_bits(state_a[$i]), ) }; } macro_rules! store { ($i:literal, $value:expr) => {{ - state_a[$i] = _mm256_extract_epi64::<0>($value) as u64; - state_b[$i] = _mm256_extract_epi64::<1>($value) as u64; - state_c[$i] = _mm256_extract_epi64::<2>($value) as u64; - state_d[$i] = _mm256_extract_epi64::<3>($value) as u64; + state_a[$i] = i64_to_u64_bits(_mm256_extract_epi64::<0>($value)); + state_b[$i] = i64_to_u64_bits(_mm256_extract_epi64::<1>($value)); + state_c[$i] = i64_to_u64_bits(_mm256_extract_epi64::<2>($value)); + state_d[$i] = i64_to_u64_bits(_mm256_extract_epi64::<3>($value)); }}; } @@ -397,7 +407,7 @@ unsafe fn keccakf_x86_avx512_x4_impl( a23 = chi4!(b23, b24, b20); a24 = chi4!(b24, b20, b21); - a0 = xor!(a0, _mm256_set1_epi64x(rc as i64)); + a0 = xor!(a0, _mm256_set1_epi64x(u64_to_i64_bits(rc))); } store!(0, a0); @@ -432,7 +442,7 @@ unsafe fn keccakf_x86_avx512_x4_impl( /// # Safety /// /// Caller must ensure `avx512f`, `avx512vl`, and `sse4.1` CPU features are available. -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", feature = "ml-kem"))] #[inline] pub(crate) unsafe fn keccakf_x86_avx512_x4( state_a: &mut [u64; 25], diff --git a/src/hashes/crypto/mod.rs b/src/hashes/crypto/mod.rs index 2c02f9e2..bee0e22f 100644 --- a/src/hashes/crypto/mod.rs +++ b/src/hashes/crypto/mod.rs @@ -1,9 +1,5 @@ //! Cryptographic hash functions. -#![cfg_attr(not(test), deny(clippy::unwrap_used))] -#![cfg_attr(not(test), deny(clippy::expect_used))] -#![cfg_attr(not(test), deny(clippy::indexing_slicing))] - #[cfg(any(feature = "blake2b", feature = "blake2s"))] use core::fmt; diff --git a/src/hashes/crypto/sha224.rs b/src/hashes/crypto/sha224.rs index 0c05e7f8..64350c42 100644 --- a/src/hashes/crypto/sha224.rs +++ b/src/hashes/crypto/sha224.rs @@ -3,8 +3,6 @@ //! SHA-224 is identical to SHA-256 except for initial hash values (H0) and //! output truncation (28 bytes / 7 words). The compression function is shared. -#![allow(clippy::indexing_slicing)] // Fixed-size arrays in finalization - use self::kernels::CompressBlocksFn; use crate::{ hashes::crypto::dispatch_util::{SizeClassDispatch, len_hint_from_u64}, @@ -181,7 +179,7 @@ impl Sha224 { compress_blocks(&mut state, &block); let mut out = [0u8; 28]; - for (chunk, &word) in out.chunks_exact_mut(4).zip(state.iter()) { + for (chunk, &word) in out.as_chunks_mut::<4>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } out @@ -196,9 +194,9 @@ impl Drop for Sha224 { } crate::traits::ct::zeroize(&mut self.block); // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.block_len, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.block_len, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } @@ -257,7 +255,7 @@ mod tests { use core::fmt::Write; let mut s = String::new(); for &b in bytes { - write!(&mut s, "{:02x}", b).unwrap(); + write!(&mut s, "{:02x}", b).expect("writing hexadecimal to String must succeed"); } s } diff --git a/src/hashes/crypto/sha224/dispatch_tables.rs b/src/hashes/crypto/sha224/dispatch_tables.rs index 181c0766..2a2924c4 100644 --- a/src/hashes/crypto/sha224/dispatch_tables.rs +++ b/src/hashes/crypto/sha224/dispatch_tables.rs @@ -2,13 +2,13 @@ //! //! Mirrors SHA-256 — same compression function, same HW accel for all sizes. -pub use super::kernels::Sha224KernelId as KernelId; +pub(crate) use super::kernels::Sha224KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -16,7 +16,7 @@ pub struct DispatchTable { pub l: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -25,7 +25,7 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_SHA_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_SHA_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Sha, s: KernelId::X86Sha, @@ -34,7 +34,7 @@ pub static X86_SHA_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "aarch64")] -pub static AARCH64_SHA2_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_SHA2_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Aarch64Sha2, s: KernelId::Aarch64Sha2, @@ -43,7 +43,7 @@ pub static AARCH64_SHA2_TABLE: DispatchTable = DispatchTable { }; #[cfg(any(target_arch = "riscv64", target_arch = "riscv32"))] -pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { +pub(crate) static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::RiscvZknh, s: KernelId::RiscvZknh, @@ -52,7 +52,7 @@ pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "wasm32")] -pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { +pub(crate) static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::WasmSimd128, s: KernelId::WasmSimd128, @@ -61,7 +61,7 @@ pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "s390x")] -pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { +pub(crate) static S390X_KIMD_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::S390xKimd, s: KernelId::S390xKimd, @@ -71,7 +71,7 @@ pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { #[cfg(target_arch = "x86_64")] { use crate::platform::caps::x86; @@ -107,5 +107,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D return &S390X_KIMD_TABLE; } } + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/sha224/kernel_test.rs b/src/hashes/crypto/sha224/kernel_test.rs index 53fb42e9..c8201594 100644 --- a/src/hashes/crypto/sha224/kernel_test.rs +++ b/src/hashes/crypto/sha224/kernel_test.rs @@ -6,13 +6,6 @@ use super::{ }; use crate::{hashes::crypto::dispatch_util::SizeClassDispatch, traits::Digest as _}; -#[allow(dead_code)] -#[derive(Clone, Debug)] -pub struct KernelResult { - pub name: &'static str, - pub digest: [u8; 28], -} - fn hasher_for_kernel(id: Sha224KernelId) -> Sha224 { let compress = compress_blocks_fn(id); Sha224 { @@ -34,41 +27,16 @@ fn digest_with_kernel(id: Sha224KernelId, data: &[u8]) -> [u8; 28] { h.finalize() } -#[must_use] -pub fn run_all_sha224_kernels(data: &[u8]) -> Vec { - let caps = crate::platform::caps(); - let mut out = Vec::with_capacity(ALL.len()); - for &id in ALL { - if caps.has(required_caps(id)) { - out.push(KernelResult { - name: id.as_str(), - digest: digest_with_kernel(id, data), - }); - } - } - out -} - -pub fn verify_sha224_kernels(data: &[u8]) -> Result<(), &'static str> { - let results = run_all_sha224_kernels(data); - let Some(first) = results.first() else { - return Ok(()); - }; - for r in &results[1..] { - if r.digest != first.digest { - return Err("sha224 kernel mismatch"); - } - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; fn pattern(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(17).wrapping_add((i >> 8) as u8)) + .map(|i| { + let bytes = i.to_le_bytes(); + bytes[0].wrapping_mul(17).wrapping_add(bytes[1]) + }) .collect() } @@ -77,7 +45,7 @@ mod tests { let caps = crate::platform::caps(); #[cfg(not(miri))] let lens = [ - 0usize, 1, 2, 3, 55, 56, 57, 63, 64, 65, 119, 120, 121, 127, 128, 129, 1000, + 0usize, 1, 2, 3, 55, 56, 57, 63, 64, 65, 119, 120, 121, 127, 128, 129, 1000, 4096, ]; #[cfg(miri)] let lens = [0usize, 1, 55, 56, 57, 63, 64, 65, 127, 128, 129]; @@ -140,10 +108,4 @@ mod tests { } } } - - #[test] - fn run_all_agree() { - verify_sha224_kernels(b"abc").expect("kernels should agree"); - verify_sha224_kernels(&pattern(4096)).expect("kernels should agree"); - } } diff --git a/src/hashes/crypto/sha224/kernels.rs b/src/hashes/crypto/sha224/kernels.rs index 3a28f3f4..13386de4 100644 --- a/src/hashes/crypto/sha224/kernels.rs +++ b/src/hashes/crypto/sha224/kernels.rs @@ -16,7 +16,7 @@ use crate::{hashes::crypto::sha256::Sha256, platform::Caps}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Sha224KernelId { +pub(crate) enum Sha224KernelId { Portable = 0, #[cfg(target_arch = "x86_64")] X86Sha = 1, @@ -34,7 +34,7 @@ impl Sha224KernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "x86_64")] @@ -52,7 +52,7 @@ impl Sha224KernelId { } #[cfg(test)] -pub const ALL: &[Sha224KernelId] = &[ +pub(crate) const ALL: &[Sha224KernelId] = &[ Sha224KernelId::Portable, #[cfg(target_arch = "x86_64")] Sha224KernelId::X86Sha, @@ -96,7 +96,7 @@ pub(crate) fn compress_blocks_fn(id: Sha224KernelId) -> CompressBlocksFn { #[inline] #[must_use] -pub const fn required_caps(id: Sha224KernelId) -> Caps { +pub(crate) const fn required_caps(id: Sha224KernelId) -> Caps { match id { Sha224KernelId::Portable => Caps::NONE, #[cfg(target_arch = "x86_64")] diff --git a/src/hashes/crypto/sha256/aarch64.rs b/src/hashes/crypto/sha256/aarch64.rs index 82fd0c87..030e216d 100644 --- a/src/hashes/crypto/sha256/aarch64.rs +++ b/src/hashes/crypto/sha256/aarch64.rs @@ -7,9 +7,6 @@ //! All functions require the `sha2` target feature. //! Callers must verify CPU capabilities before calling. -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] - #[cfg(target_arch = "aarch64")] use core::arch::aarch64::*; @@ -258,19 +255,19 @@ pub(crate) unsafe fn compress_blocks_aarch64_sha2(state: &mut [u32; 8], blocks: efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); s1 = vsha256su1q_u32(vsha256su0q_u32(s1, s2), s3, s0); - tmp = vaddq_u32(s1, vld1q_u32(kp.add(t + 4))); + tmp = vaddq_u32(s1, vld1q_u32(kp.add(t.strict_add(4)))); abcd_prev = abcd; abcd = vsha256hq_u32(abcd_prev, efgh, tmp); efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); s2 = vsha256su1q_u32(vsha256su0q_u32(s2, s3), s0, s1); - tmp = vaddq_u32(s2, vld1q_u32(kp.add(t + 8))); + tmp = vaddq_u32(s2, vld1q_u32(kp.add(t.strict_add(8)))); abcd_prev = abcd; abcd = vsha256hq_u32(abcd_prev, efgh, tmp); efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); s3 = vsha256su1q_u32(vsha256su0q_u32(s3, s0), s1, s2); - tmp = vaddq_u32(s3, vld1q_u32(kp.add(t + 12))); + tmp = vaddq_u32(s3, vld1q_u32(kp.add(t.strict_add(12)))); abcd_prev = abcd; abcd = vsha256hq_u32(abcd_prev, efgh, tmp); efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); diff --git a/src/hashes/crypto/sha256/dispatch_tables.rs b/src/hashes/crypto/sha256/dispatch_tables.rs index d28808d1..06a5d45e 100644 --- a/src/hashes/crypto/sha256/dispatch_tables.rs +++ b/src/hashes/crypto/sha256/dispatch_tables.rs @@ -6,13 +6,13 @@ //! POWER uses portable compression because the retired hybrid `vshasigmaw` //! kernel lost to portable Rust on POWER10. -pub use super::kernels::Sha256KernelId as KernelId; +pub(crate) use super::kernels::Sha256KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -20,7 +20,7 @@ pub struct DispatchTable { pub l: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -29,7 +29,7 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_SHA_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_SHA_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Sha, s: KernelId::X86Sha, @@ -38,7 +38,7 @@ pub static X86_SHA_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "aarch64")] -pub static AARCH64_SHA2_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_SHA2_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Aarch64Sha2, s: KernelId::Aarch64Sha2, @@ -47,7 +47,7 @@ pub static AARCH64_SHA2_TABLE: DispatchTable = DispatchTable { }; #[cfg(any(target_arch = "riscv64", target_arch = "riscv32"))] -pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { +pub(crate) static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::RiscvZknh, s: KernelId::RiscvZknh, @@ -56,7 +56,7 @@ pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "wasm32")] -pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { +pub(crate) static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::WasmSimd128, s: KernelId::WasmSimd128, @@ -65,7 +65,7 @@ pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "s390x")] -pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { +pub(crate) static S390X_KIMD_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::S390xKimd, s: KernelId::S390xKimd, @@ -75,7 +75,7 @@ pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { #[cfg(target_arch = "x86_64")] { use crate::platform::caps::x86; @@ -111,5 +111,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D return &S390X_KIMD_TABLE; } } + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/sha256/kernel_test.rs b/src/hashes/crypto/sha256/kernel_test.rs index 7df8f1ce..0394d050 100644 --- a/src/hashes/crypto/sha256/kernel_test.rs +++ b/src/hashes/crypto/sha256/kernel_test.rs @@ -6,13 +6,6 @@ use super::{ }; use crate::hashes::crypto::dispatch_util::SizeClassDispatch; -#[derive(Clone, Debug)] -#[allow(dead_code)] -pub struct KernelResult { - pub name: &'static str, - pub digest: [u8; 32], -} - fn hasher_for_kernel(id: Sha256KernelId) -> Sha256 { let compress = compress_blocks_fn(id); Sha256 { @@ -34,41 +27,16 @@ fn digest_with_kernel(id: Sha256KernelId, data: &[u8]) -> [u8; 32] { h.finalize() } -#[must_use] -pub fn run_all_sha256_kernels(data: &[u8]) -> Vec { - let caps = crate::platform::caps(); - let mut out = Vec::with_capacity(ALL.len()); - for &id in ALL { - if caps.has(required_caps(id)) { - out.push(KernelResult { - name: id.as_str(), - digest: digest_with_kernel(id, data), - }); - } - } - out -} - -pub fn verify_sha256_kernels(data: &[u8]) -> Result<(), &'static str> { - let results = run_all_sha256_kernels(data); - let Some(first) = results.first() else { - return Ok(()); - }; - for r in &results[1..] { - if r.digest != first.digest { - return Err("sha256 kernel mismatch"); - } - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; fn pattern(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(31).wrapping_add((i >> 8) as u8)) + .map(|i| { + let bytes = i.to_le_bytes(); + bytes[0].wrapping_mul(31).wrapping_add(bytes[1]) + }) .collect() } @@ -91,7 +59,7 @@ mod tests { #[cfg(not(miri))] let lens = [ - 0usize, 1, 2, 3, 55, 56, 57, 63, 64, 65, 119, 120, 121, 127, 128, 129, 1000, + 0usize, 1, 2, 3, 55, 56, 57, 63, 64, 65, 119, 120, 121, 127, 128, 129, 1000, 4096, ]; #[cfg(miri)] let lens = [0usize, 1, 55, 56, 57, 63, 64, 65, 127, 128, 129]; @@ -152,12 +120,6 @@ mod tests { } } - #[test] - fn run_all_agree() { - verify_sha256_kernels(b"abc").expect("kernels should agree"); - verify_sha256_kernels(&pattern(4096)).expect("kernels should agree"); - } - #[test] #[cfg(target_arch = "x86_64")] fn x86_sha_kernel_requires_sha_and_sse41() { diff --git a/src/hashes/crypto/sha256/kernels.rs b/src/hashes/crypto/sha256/kernels.rs index 25482e79..cf761bf3 100644 --- a/src/hashes/crypto/sha256/kernels.rs +++ b/src/hashes/crypto/sha256/kernels.rs @@ -16,7 +16,7 @@ pub(crate) type CompressBlocksFn = fn(&mut [u32; 8], &[u8]); #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Sha256KernelId { +pub(crate) enum Sha256KernelId { Portable = 0, #[cfg(target_arch = "x86_64")] X86Sha = 1, @@ -34,7 +34,7 @@ impl Sha256KernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "x86_64")] @@ -52,7 +52,7 @@ impl Sha256KernelId { } #[cfg(test)] -pub const ALL: &[Sha256KernelId] = &[ +pub(crate) const ALL: &[Sha256KernelId] = &[ Sha256KernelId::Portable, #[cfg(target_arch = "x86_64")] Sha256KernelId::X86Sha, @@ -120,7 +120,7 @@ pub(crate) fn compress_blocks_fn(id: Sha256KernelId) -> CompressBlocksFn { #[inline] #[must_use] -pub const fn required_caps(id: Sha256KernelId) -> Caps { +pub(crate) const fn required_caps(id: Sha256KernelId) -> Caps { match id { Sha256KernelId::Portable => Caps::NONE, #[cfg(target_arch = "x86_64")] @@ -171,11 +171,11 @@ pub(crate) const COMPILE_TIME_HW: bool = cfg!(not(miri)) pub(crate) fn compile_time_best() -> CompressBlocksFn { #[cfg(miri)] { - return Sha256::compress_blocks_portable; + Sha256::compress_blocks_portable } #[cfg(all(not(miri), target_arch = "x86_64", target_feature = "sha", target_feature = "sse4.1"))] { - return compress_blocks_x86_sha; + compress_blocks_x86_sha } #[cfg(all( not(miri), @@ -183,7 +183,7 @@ pub(crate) fn compile_time_best() -> CompressBlocksFn { any(target_os = "macos", target_feature = "sha2") ))] { - return compress_blocks_aarch64_sha2; + compress_blocks_aarch64_sha2 } #[cfg(all( not(miri), @@ -191,18 +191,28 @@ pub(crate) fn compile_time_best() -> CompressBlocksFn { target_feature = "zknh" ))] { - return compress_blocks_riscv_zknh; + compress_blocks_riscv_zknh } #[cfg(all(not(miri), target_arch = "wasm32", target_feature = "simd128"))] { - return compress_blocks_wasm_simd128; + compress_blocks_wasm_simd128 + } + #[cfg(all( + not(miri), + not(any( + all(target_arch = "x86_64", target_feature = "sha", target_feature = "sse4.1"), + all(target_arch = "aarch64", any(target_os = "macos", target_feature = "sha2")), + all(any(target_arch = "riscv64", target_arch = "riscv32"), target_feature = "zknh"), + all(target_arch = "wasm32", target_feature = "simd128") + )) + ))] + { + Sha256::compress_blocks_portable } - #[allow(unreachable_code)] - Sha256::compress_blocks_portable } /// Kernel name for the compile-time-best path (introspection). -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] pub(crate) const COMPILE_TIME_NAME: &str = if cfg!(miri) { "portable" } else if cfg!(all( diff --git a/src/hashes/crypto/sha256/mod.rs b/src/hashes/crypto/sha256/mod.rs index c67d238d..5a24ba47 100644 --- a/src/hashes/crypto/sha256/mod.rs +++ b/src/hashes/crypto/sha256/mod.rs @@ -1,7 +1,5 @@ //! SHA-256 (FIPS 180-4). -#![allow(clippy::indexing_slicing)] // Fixed-size arrays + compression schedule - use self::kernels::CompressBlocksFn; use crate::{ hashes::{ @@ -387,7 +385,7 @@ impl Sha256Prefix { unsafe { core::ptr::write_volatile(word, 0) }; } // SAFETY: bytes_hashed is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } @@ -453,14 +451,11 @@ impl Sha256 { #[inline] pub(crate) fn compress_blocks_portable(state: &mut [u32; 8], blocks: &[u8]) { - debug_assert_eq!(blocks.len() % BLOCK_LEN, 0); - let mut chunks = blocks.chunks_exact(BLOCK_LEN); - for chunk in &mut chunks { - // SAFETY: `chunks_exact(BLOCK_LEN)` yields slices of exactly `BLOCK_LEN` bytes. - let block = unsafe { &*(chunk.as_ptr() as *const [u8; BLOCK_LEN]) }; + let (chunks, remainder) = blocks.as_chunks::(); + for block in chunks { Self::compress_block(state, block); } - debug_assert!(chunks.remainder().is_empty()); + debug_assert!(remainder.is_empty()); } #[inline] @@ -622,7 +617,7 @@ impl Sha256 { compress_blocks(&mut state, &block); let mut out = [0u8; 32]; - for (chunk, &word) in out.chunks_exact_mut(4).zip(state.iter()) { + for (chunk, &word) in out.as_chunks_mut::<4>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } if ZEROIZE { @@ -703,9 +698,8 @@ impl Sha256 { #[cfg(all(feature = "hmac", not(target_arch = "x86_64")))] fn reset_update_mode_to_aligned_prefix(&mut self, _prefix: Sha256Prefix) {} - #[cfg(all(feature = "hmac", any(test, feature = "diag")))] + #[cfg(all(feature = "hmac", any(feature = "diag", all(test, feature = "hkdf"))))] #[inline] - #[allow(dead_code)] pub(crate) fn new_with_compress_for_test(compress_blocks: CompressBlocksFn) -> Self { Self { state: H0, @@ -744,9 +738,9 @@ impl Drop for Sha256 { } crate::traits::ct::zeroize(&mut self.block); // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.block_len, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.block_len, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } @@ -794,7 +788,7 @@ mod tests { use core::fmt::Write; let mut s = String::new(); for &b in bytes { - write!(&mut s, "{:02x}", b).unwrap(); + write!(&mut s, "{:02x}", b).expect("writing hexadecimal to String must succeed"); } s } diff --git a/src/hashes/crypto/sha256/riscv64.rs b/src/hashes/crypto/sha256/riscv64.rs index 12ee02ac..c47798c4 100644 --- a/src/hashes/crypto/sha256/riscv64.rs +++ b/src/hashes/crypto/sha256/riscv64.rs @@ -8,8 +8,6 @@ //! //! All functions require the `zknh` target feature. -#![allow(clippy::indexing_slicing)] // Fixed-size arrays + compression schedule - #[cfg(target_arch = "riscv32")] use core::arch::riscv32::{sha256sig0, sha256sig1, sha256sum0, sha256sum1}; #[cfg(target_arch = "riscv64")] @@ -17,10 +15,6 @@ use core::arch::riscv64::{sha256sig0, sha256sig1, sha256sum0, sha256sum1}; use super::BLOCK_LEN; -// Safe wrappers for the unsafe Zknh intrinsics. -// SAFETY: only called from within `compress_blocks_zknh` which has -// `#[target_feature(enable = "zknh")]`, guaranteeing the feature is available. - #[inline(always)] fn sum0(x: u32) -> u32 { // SAFETY: `compress_blocks_zknh` calls this wrapper only inside its Zknh target-feature scope. @@ -52,12 +46,9 @@ fn sig1(x: u32) -> u32 { /// Caller must ensure `zknh` CPU feature is available. #[target_feature(enable = "zknh")] pub(crate) unsafe fn compress_blocks_zknh(state: &mut [u32; 8], blocks: &[u8]) { - debug_assert_eq!(blocks.len() % BLOCK_LEN, 0); - let mut chunks = blocks.chunks_exact(BLOCK_LEN); - for chunk in &mut chunks { - // SAFETY: `chunks_exact(BLOCK_LEN)` yields slices of exactly `BLOCK_LEN` bytes. - let block = unsafe { &*(chunk.as_ptr() as *const [u8; BLOCK_LEN]) }; + let (chunks, remainder) = blocks.as_chunks::(); + debug_assert!(remainder.is_empty()); + for block in chunks { super::compress_block_with(state, block, sum0, sum1, sig0, sig1); } - debug_assert!(chunks.remainder().is_empty()); } diff --git a/src/hashes/crypto/sha256/s390x.rs b/src/hashes/crypto/sha256/s390x.rs index 9f14fc3f..21dbf7c7 100644 --- a/src/hashes/crypto/sha256/s390x.rs +++ b/src/hashes/crypto/sha256/s390x.rs @@ -13,9 +13,6 @@ //! //! Requires the MSA facility (CPACF). Caller must verify `s390x::MSA`. -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] - use super::BLOCK_LEN; /// SHA-256 block compression via KIMD instruction. diff --git a/src/hashes/crypto/sha256/wasm.rs b/src/hashes/crypto/sha256/wasm.rs index a074102e..ff4a1e15 100644 --- a/src/hashes/crypto/sha256/wasm.rs +++ b/src/hashes/crypto/sha256/wasm.rs @@ -3,8 +3,6 @@ //! Vectorizes the message schedule computation using 128-bit SIMD (4 × u32 //! lanes). Compression rounds remain scalar (sequential data dependency). -#![allow(clippy::indexing_slicing)] // Fixed-size arrays + compression schedule - #[cfg(target_arch = "wasm32")] use core::arch::wasm32::*; @@ -12,12 +10,20 @@ use super::{BLOCK_LEN, K, ch, maj}; use crate::hashes::util::rotr32; /// Load 4 big-endian message words from `ptr` into a v128, byte-swapping each. +/// +/// # Safety +/// +/// `ptr` must be valid for an unaligned 16-byte read. The caller must execute +/// this function only when WebAssembly SIMD128 is enabled. #[cfg(target_arch = "wasm32")] #[inline(always)] unsafe fn load_be(ptr: *const u8) -> v128 { - // SAFETY: caller guarantees `ptr` is valid for a 16-byte aligned read - // and the simd128 target feature is enabled. - let raw = unsafe { v128_load(ptr as *const v128) }; + // SAFETY: the caller guarantees that `ptr..ptr+16` is readable. The byte + // array has alignment one, and `read_unaligned` accepts any address. + let bytes = unsafe { ptr.cast::<[u8; 16]>().read_unaligned() }; + // SAFETY: `[u8; 16]` and `v128` have the same size, and every SIMD bit + // pattern is valid. + let raw = unsafe { core::mem::transmute::<[u8; 16], v128>(bytes) }; i8x16_shuffle::<3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12>(raw, raw) } @@ -68,36 +74,36 @@ fn schedule_4(w: &mut [v128; 16], i: usize) { let t = i << 2; let mut slot = u32x4_splat(0); - let w16_0 = schedule_word(w, t - 16); - let w15_0 = schedule_word(w, t - 15); - let w7_0 = schedule_word(w, t - 7); - let w2_0 = schedule_word(w, t - 2); + let w16_0 = schedule_word(w, t.strict_sub(16)); + let w15_0 = schedule_word(w, t.strict_sub(15)); + let w7_0 = schedule_word(w, t.strict_sub(7)); + let w2_0 = schedule_word(w, t.strict_sub(2)); let w0 = small_sigma1(w2_0) .wrapping_add(w7_0) .wrapping_add(small_sigma0(w15_0)) .wrapping_add(w16_0); - let w16_1 = schedule_word(w, t - 15); - let w15_1 = schedule_word(w, t - 14); - let w7_1 = schedule_word(w, t - 6); - let w2_1 = schedule_word(w, t - 1); + let w16_1 = schedule_word(w, t.strict_sub(15)); + let w15_1 = schedule_word(w, t.strict_sub(14)); + let w7_1 = schedule_word(w, t.strict_sub(6)); + let w2_1 = schedule_word(w, t.strict_sub(1)); let w1 = small_sigma1(w2_1) .wrapping_add(w7_1) .wrapping_add(small_sigma0(w15_1)) .wrapping_add(w16_1); - let w16_2 = schedule_word(w, t - 14); - let w15_2 = schedule_word(w, t - 13); - let w7_2 = schedule_word(w, t - 5); + let w16_2 = schedule_word(w, t.strict_sub(14)); + let w15_2 = schedule_word(w, t.strict_sub(13)); + let w7_2 = schedule_word(w, t.strict_sub(5)); let w2_2 = w0; let w2 = small_sigma1(w2_2) .wrapping_add(w7_2) .wrapping_add(small_sigma0(w15_2)) .wrapping_add(w16_2); - let w16_3 = schedule_word(w, t - 13); - let w15_3 = schedule_word(w, t - 12); - let w7_3 = schedule_word(w, t - 4); + let w16_3 = schedule_word(w, t.strict_sub(13)); + let w15_3 = schedule_word(w, t.strict_sub(12)); + let w7_3 = schedule_word(w, t.strict_sub(4)); let w2_3 = w1; let w3 = small_sigma1(w2_3) .wrapping_add(w7_3) @@ -117,6 +123,11 @@ fn schedule_4(w: &mut [v128; 16], i: usize) { /// /// The message schedule is computed with SIMD (4 words per v128). /// Compression rounds are scalar (sequential dependency chain). +/// +/// # Safety +/// +/// The caller must ensure WebAssembly SIMD128 is available and `blocks` +/// contains only complete SHA-256 blocks. #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] pub(crate) unsafe fn compress_blocks_wasm_simd(state: &mut [u32; 8], blocks: &[u8]) { @@ -202,9 +213,9 @@ pub(crate) unsafe fn compress_blocks_wasm_simd(state: &mut [u32; 8], blocks: &[u schedule_4(&mut wv, r / 4); let sched = wv[(r / 4) & 0xF]; sha_round!(K[r], u32x4_extract_lane::<0>(sched)); - sha_round!(K[r + 1], u32x4_extract_lane::<1>(sched)); - sha_round!(K[r + 2], u32x4_extract_lane::<2>(sched)); - sha_round!(K[r + 3], u32x4_extract_lane::<3>(sched)); + sha_round!(K[r.strict_add(1)], u32x4_extract_lane::<1>(sched)); + sha_round!(K[r.strict_add(2)], u32x4_extract_lane::<2>(sched)); + sha_round!(K[r.strict_add(3)], u32x4_extract_lane::<3>(sched)); } state[0] = state[0].wrapping_add(a); diff --git a/src/hashes/crypto/sha256/x86_64.rs b/src/hashes/crypto/sha256/x86_64.rs index 6aa3469d..6f2e426a 100644 --- a/src/hashes/crypto/sha256/x86_64.rs +++ b/src/hashes/crypto/sha256/x86_64.rs @@ -8,9 +8,6 @@ //! All functions require `sha` and `sse4.1` target features. //! Callers must verify CPU capabilities before calling. -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] - #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; diff --git a/src/hashes/crypto/sha3.rs b/src/hashes/crypto/sha3.rs index b71a3b4e..ca02f464 100644 --- a/src/hashes/crypto/sha3.rs +++ b/src/hashes/crypto/sha3.rs @@ -75,26 +75,17 @@ impl MlKemShake256XofReader { } #[inline] - #[allow(clippy::too_many_arguments)] - pub(crate) fn squeeze_quad( - a: &mut Self, + pub(crate) fn squeeze_quad( + &mut self, b: &mut Self, c: &mut Self, d: &mut Self, - out_a: &mut [u8], - out_b: &mut [u8], - out_c: &mut [u8], - out_d: &mut [u8], + outputs: &mut [[u8; N]; 4], ) { + let [out_a, out_b, out_c, out_d] = outputs; KeccakXof::<136>::squeeze_quad_into( - &mut a.inner, - &mut b.inner, - &mut c.inner, - &mut d.inner, - out_a, - out_b, - out_c, - out_d, + [&mut self.inner, &mut b.inner, &mut c.inner, &mut d.inner], + [out_a, out_b, out_c, out_d], ); } } @@ -148,13 +139,9 @@ pub fn diag_zeroize_mlkem_shake256_pair(mut seed: [u8; 32]) -> u8 { pub fn diag_zeroize_mlkem_shake256_quad(mut seed: [u8; 32]) -> u8 { let (mut a, mut b, mut c, mut d) = MlKemShake256XofReader::seeded_32_1_quad(&seed, 1, 2, 3, 4); crate::traits::ct::zeroize(&mut seed); - let mut out_a = [0u8; 192]; - let mut out_b = [0u8; 192]; - let mut out_c = [0u8; 192]; - let mut out_d = [0u8; 192]; - MlKemShake256XofReader::squeeze_quad( - &mut a, &mut b, &mut c, &mut d, &mut out_a, &mut out_b, &mut out_c, &mut out_d, - ); + let mut outputs = [[0u8; 192]; 4]; + a.squeeze_quad(&mut b, &mut c, &mut d, &mut outputs); + let [mut out_a, mut out_b, mut out_c, mut out_d] = outputs; let observed = out_a[0] ^ out_b[0] ^ out_c[0] ^ out_d[0]; crate::traits::ct::zeroize(&mut out_a); crate::traits::ct::zeroize(&mut out_b); @@ -497,12 +484,14 @@ impl core::fmt::Debug for Shake128 { } impl Shake128 { + /// Create an empty SHAKE128 state. #[inline] #[must_use] pub fn new() -> Self { Self::default() } + /// Absorb `data` and return a SHAKE128 output reader. #[inline] #[must_use] pub fn xof(data: &[u8]) -> Shake128XofReader { @@ -511,11 +500,13 @@ impl Shake128 { h.finalize_xof() } + /// Absorb more input into this state. #[inline] pub fn update(&mut self, data: &[u8]) { self.core.update(data); } + /// Finalize absorption and return an output reader. #[inline] #[must_use] pub fn finalize_xof(self) -> Shake128XofReader { @@ -524,6 +515,7 @@ impl Shake128 { } } + /// Reset this state to its empty value. #[inline] pub fn reset(&mut self) { *self = Self::default(); @@ -619,6 +611,7 @@ impl Shake128 { } } +/// Streaming reader for SHAKE128 output. #[derive(Clone)] pub struct Shake128XofReader { inner: PublicKeccakXof<168>, @@ -678,26 +671,17 @@ impl Shake128XofReader { )) ) ))] - #[allow(clippy::too_many_arguments)] - pub(crate) fn squeeze_quad( - a: &mut Self, + pub(crate) fn squeeze_quad( + &mut self, b: &mut Self, c: &mut Self, d: &mut Self, - out_a: &mut [u8], - out_b: &mut [u8], - out_c: &mut [u8], - out_d: &mut [u8], + outputs: &mut [[u8; N]; 4], ) { + let [out_a, out_b, out_c, out_d] = outputs; PublicKeccakXof::<168>::squeeze_quad_into( - &mut a.inner, - &mut b.inner, - &mut c.inner, - &mut d.inner, - out_a, - out_b, - out_c, - out_d, + [&mut self.inner, &mut b.inner, &mut c.inner, &mut d.inner], + [out_a, out_b, out_c, out_d], ); } @@ -740,12 +724,14 @@ impl Shake128XofReader { impl_xof_read!(Shake128XofReader); impl Shake256 { + /// Create an empty SHAKE256 state. #[inline] #[must_use] pub fn new() -> Self { Self::default() } + /// Absorb `data` and return a SHAKE256 output reader. #[inline] #[must_use] pub fn xof(data: &[u8]) -> Shake256XofReader { @@ -754,11 +740,13 @@ impl Shake256 { h.finalize_xof() } + /// Absorb more input into this state. #[inline] pub fn update(&mut self, data: &[u8]) { self.core.update(data); } + /// Finalize absorption and return an output reader. #[inline] #[must_use] pub fn finalize_xof(self) -> Shake256XofReader { @@ -767,6 +755,7 @@ impl Shake256 { } } + /// Reset this state to its empty value. #[inline] pub fn reset(&mut self) { *self = Self::default(); @@ -782,6 +771,7 @@ impl Shake256 { } } +/// Streaming reader for SHAKE256 output. #[derive(Clone)] pub struct Shake256XofReader { inner: PublicKeccakXof<136>, @@ -804,8 +794,6 @@ impl_xof_read!(Shake256XofReader); #[cfg(test)] mod tests { - #[cfg(feature = "ml-kem")] - use super::Shake128XofReader; use super::{Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256}; use crate::traits::{Digest, Xof}; @@ -814,7 +802,7 @@ mod tests { use core::fmt::Write; let mut s = String::new(); for &b in bytes { - write!(&mut s, "{:02x}", b).unwrap(); + write!(&mut s, "{b:02x}").expect("writing to a String should not fail"); } s } @@ -881,30 +869,8 @@ mod tests { let msg_d = &[0xa5u8; 409]; let (mut quad_a, mut quad_b, mut quad_c, mut quad_d) = Shake128::xof_quad(msg_a, msg_b, msg_c, msg_d); - let mut actual_a = [0u8; 320]; - let mut actual_b = [0u8; 320]; - let mut actual_c = [0u8; 320]; - let mut actual_d = [0u8; 320]; - Shake128XofReader::squeeze_quad( - &mut quad_a, - &mut quad_b, - &mut quad_c, - &mut quad_d, - &mut actual_a[..168], - &mut actual_b[..168], - &mut actual_c[..168], - &mut actual_d[..168], - ); - Shake128XofReader::squeeze_quad( - &mut quad_a, - &mut quad_b, - &mut quad_c, - &mut quad_d, - &mut actual_a[168..], - &mut actual_b[168..], - &mut actual_c[168..], - &mut actual_d[168..], - ); + let mut actual = [[0u8; 320]; 4]; + quad_a.squeeze_quad(&mut quad_b, &mut quad_c, &mut quad_d, &mut actual); let mut expected_a = [0u8; 320]; let mut expected_b = [0u8; 320]; @@ -915,10 +881,10 @@ mod tests { Shake128::xof(msg_c).squeeze(&mut expected_c); Shake128::xof(msg_d).squeeze(&mut expected_d); - assert_eq!(actual_a, expected_a, "lane 0"); - assert_eq!(actual_b, expected_b, "lane 1"); - assert_eq!(actual_c, expected_c, "lane 2"); - assert_eq!(actual_d, expected_d, "lane 3"); + assert_eq!(actual[0], expected_a, "lane 0"); + assert_eq!(actual[1], expected_b, "lane 1"); + assert_eq!(actual[2], expected_c, "lane 2"); + assert_eq!(actual[3], expected_d, "lane 3"); } /// Test inputs of length `RATE - 1` for each SHA-3 variant. diff --git a/src/hashes/crypto/sha384.rs b/src/hashes/crypto/sha384.rs index 53720dea..44b1feb2 100644 --- a/src/hashes/crypto/sha384.rs +++ b/src/hashes/crypto/sha384.rs @@ -3,8 +3,6 @@ //! SHA-384 is identical to SHA-512 except for initial hash values (H0) and //! output truncation (48 bytes / 6 words). The compression function is shared. -#![allow(clippy::indexing_slicing)] // Fixed-size arrays in finalization - use self::kernels::CompressBlocksFn; use super::sha512::Sha512; use crate::{ @@ -82,7 +80,7 @@ impl Sha384Prefix { unsafe { core::ptr::write_volatile(word, 0) }; } // SAFETY: bytes_hashed is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } @@ -255,7 +253,6 @@ impl Sha384 { #[cfg(all(feature = "hmac", any(test, feature = "diag")))] #[inline] - #[allow(dead_code)] pub(crate) fn new_with_compress_for_test(compress_blocks: CompressBlocksFn) -> Self { Self { state: H0, @@ -282,9 +279,9 @@ impl Drop for Sha384 { } crate::traits::ct::zeroize(&mut self.block); // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.block_len, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.block_len, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } diff --git a/src/hashes/crypto/sha384/dispatch_tables.rs b/src/hashes/crypto/sha384/dispatch_tables.rs index 90de8339..e2eae2e7 100644 --- a/src/hashes/crypto/sha384/dispatch_tables.rs +++ b/src/hashes/crypto/sha384/dispatch_tables.rs @@ -3,13 +3,13 @@ //! SHA-384 uses identical compression to SHA-512, so the same hardware kernels //! and cascade order apply. -pub use super::kernels::Sha384KernelId as KernelId; +pub(crate) use super::kernels::Sha384KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -17,7 +17,7 @@ pub struct DispatchTable { pub l: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -26,7 +26,7 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "aarch64")] -pub static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Aarch64Sha512, s: KernelId::Aarch64Sha512, @@ -35,7 +35,7 @@ pub static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_SHA512_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_SHA512_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Sha512, s: KernelId::X86Sha512, @@ -44,7 +44,7 @@ pub static X86_SHA512_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx512vl, s: KernelId::X86Avx512vl, @@ -53,7 +53,7 @@ pub static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx2Decoupled, s: KernelId::X86Avx2Decoupled, @@ -62,7 +62,7 @@ pub static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx512vlDecoupled, s: KernelId::X86Avx512vlDecoupled, @@ -71,7 +71,7 @@ pub static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "riscv64")] -pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { +pub(crate) static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Riscv64Zknh, s: KernelId::Riscv64Zknh, @@ -80,7 +80,7 @@ pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "wasm32")] -pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { +pub(crate) static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::WasmSimd128, s: KernelId::WasmSimd128, @@ -89,7 +89,7 @@ pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "s390x")] -pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { +pub(crate) static S390X_KIMD_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::S390xKimd, s: KernelId::S390xKimd, @@ -99,7 +99,7 @@ pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { // x86_64 cascade: SHA-512 NI > vendor-aware AVX2/AVX-512VL > Portable // AMD: AVX2 decoupled > AVX-512VL; Intel: AVX-512VL decoupled > AVX2 decoupled. // See sha512/dispatch_tables.rs for full rationale. @@ -157,5 +157,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D return &S390X_KIMD_TABLE; } } + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/sha384/kernel_test.rs b/src/hashes/crypto/sha384/kernel_test.rs index fe5a45f0..798388a3 100644 --- a/src/hashes/crypto/sha384/kernel_test.rs +++ b/src/hashes/crypto/sha384/kernel_test.rs @@ -6,13 +6,6 @@ use super::{ }; use crate::{hashes::crypto::dispatch_util::SizeClassDispatch, traits::Digest as _}; -#[allow(dead_code)] -#[derive(Clone, Debug)] -pub struct KernelResult { - pub name: &'static str, - pub digest: [u8; 48], -} - fn hasher_for_kernel(id: Sha384KernelId) -> Sha384 { let compress = compress_blocks_fn(id); Sha384 { @@ -63,43 +56,16 @@ fn digest_oneshot_with_kernel(id: Sha384KernelId, data: &[u8]) -> [u8; 48] { out } -#[allow(dead_code)] -#[must_use] -pub fn run_all_sha384_kernels(data: &[u8]) -> Vec { - let caps = crate::platform::caps(); - let mut out = Vec::with_capacity(ALL.len()); - for &id in ALL { - if caps.has(required_caps(id)) { - out.push(KernelResult { - name: id.as_str(), - digest: digest_with_kernel(id, data), - }); - } - } - out -} - -#[allow(dead_code)] -pub fn verify_sha384_kernels(data: &[u8]) -> Result<(), &'static str> { - let results = run_all_sha384_kernels(data); - let Some(first) = results.first() else { - return Ok(()); - }; - for r in &results[1..] { - if r.digest != first.digest { - return Err("sha384 kernel mismatch"); - } - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; fn pattern(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(19).wrapping_add((i >> 8) as u8)) + .map(|i| { + let bytes = i.to_le_bytes(); + bytes[0].wrapping_mul(19).wrapping_add(bytes[1]) + }) .collect() } @@ -108,7 +74,7 @@ mod tests { let caps = crate::platform::caps(); #[cfg(not(miri))] let lens = [ - 0usize, 1, 2, 3, 111, 112, 113, 127, 128, 129, 239, 240, 241, 255, 256, 257, 1000, + 0usize, 1, 2, 3, 111, 112, 113, 127, 128, 129, 239, 240, 241, 255, 256, 257, 1000, 4096, ]; #[cfg(miri)] let lens = [0usize, 1, 111, 112, 113, 127, 128, 129, 255, 256, 257]; diff --git a/src/hashes/crypto/sha384/kernels.rs b/src/hashes/crypto/sha384/kernels.rs index eef50d09..4ef7b0e2 100644 --- a/src/hashes/crypto/sha384/kernels.rs +++ b/src/hashes/crypto/sha384/kernels.rs @@ -6,7 +6,7 @@ use crate::{hashes::crypto::sha512::Sha512, platform::Caps}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Sha384KernelId { +pub(crate) enum Sha384KernelId { Portable = 0, #[cfg(target_arch = "aarch64")] Aarch64Sha512 = 1, @@ -30,7 +30,7 @@ impl Sha384KernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "aarch64")] @@ -110,13 +110,13 @@ pub(crate) fn compress_blocks_fn(id: Sha384KernelId) -> CompressBlocksFn { #[inline] #[must_use] -pub const fn required_caps(id: Sha384KernelId) -> Caps { +pub(crate) const fn required_caps(id: Sha384KernelId) -> Caps { crate::hashes::crypto::sha512::kernels::required_caps(to_sha512_kernel_id(id)) } // Keep kernel tests focused on backends that runtime dispatch can actually pick. #[cfg(test)] -pub const ALL: &[Sha384KernelId] = &[ +pub(crate) const ALL: &[Sha384KernelId] = &[ Sha384KernelId::Portable, #[cfg(target_arch = "aarch64")] Sha384KernelId::Aarch64Sha512, diff --git a/src/hashes/crypto/sha512/aarch64.rs b/src/hashes/crypto/sha512/aarch64.rs index d36f94c8..dd185705 100644 --- a/src/hashes/crypto/sha512/aarch64.rs +++ b/src/hashes/crypto/sha512/aarch64.rs @@ -6,10 +6,6 @@ //! //! Available on Apple Silicon (M1+), Graviton2+, Ampere Altra, Cortex-A76+. -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::indexing_slicing)] - #[cfg(target_arch = "aarch64")] use core::arch::aarch64::*; @@ -163,49 +159,49 @@ pub(crate) unsafe fn compress_blocks_aarch64_sha512(state: &mut [u64; 8], blocks cd = vaddq_u64(cd, intermed); s1 = vsha512su1q_u64(vsha512su0q_u64(s1, s2), s0, vextq_u64(s5, s6, 1)); - initial_sum = vaddq_u64(s1, k(pair + 1)); + initial_sum = vaddq_u64(s1, k(pair.strict_add(1))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ef); intermed = vsha512hq_u64(sum, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); ef = vsha512h2q_u64(intermed, ab, gh); ab = vaddq_u64(ab, intermed); s2 = vsha512su1q_u64(vsha512su0q_u64(s2, s3), s1, vextq_u64(s6, s7, 1)); - initial_sum = vaddq_u64(s2, k(pair + 2)); + initial_sum = vaddq_u64(s2, k(pair.strict_add(2))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), cd); intermed = vsha512hq_u64(sum, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); cd = vsha512h2q_u64(intermed, gh, ef); gh = vaddq_u64(gh, intermed); s3 = vsha512su1q_u64(vsha512su0q_u64(s3, s4), s2, vextq_u64(s7, s0, 1)); - initial_sum = vaddq_u64(s3, k(pair + 3)); + initial_sum = vaddq_u64(s3, k(pair.strict_add(3))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ab); intermed = vsha512hq_u64(sum, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); ab = vsha512h2q_u64(intermed, ef, cd); ef = vaddq_u64(ef, intermed); s4 = vsha512su1q_u64(vsha512su0q_u64(s4, s5), s3, vextq_u64(s0, s1, 1)); - initial_sum = vaddq_u64(s4, k(pair + 4)); + initial_sum = vaddq_u64(s4, k(pair.strict_add(4))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), gh); intermed = vsha512hq_u64(sum, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); gh = vsha512h2q_u64(intermed, cd, ab); cd = vaddq_u64(cd, intermed); s5 = vsha512su1q_u64(vsha512su0q_u64(s5, s6), s4, vextq_u64(s1, s2, 1)); - initial_sum = vaddq_u64(s5, k(pair + 5)); + initial_sum = vaddq_u64(s5, k(pair.strict_add(5))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ef); intermed = vsha512hq_u64(sum, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); ef = vsha512h2q_u64(intermed, ab, gh); ab = vaddq_u64(ab, intermed); s6 = vsha512su1q_u64(vsha512su0q_u64(s6, s7), s5, vextq_u64(s2, s3, 1)); - initial_sum = vaddq_u64(s6, k(pair + 6)); + initial_sum = vaddq_u64(s6, k(pair.strict_add(6))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), cd); intermed = vsha512hq_u64(sum, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); cd = vsha512h2q_u64(intermed, gh, ef); gh = vaddq_u64(gh, intermed); s7 = vsha512su1q_u64(vsha512su0q_u64(s7, s0), s6, vextq_u64(s3, s4, 1)); - initial_sum = vaddq_u64(s7, k(pair + 7)); + initial_sum = vaddq_u64(s7, k(pair.strict_add(7))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ab); intermed = vsha512hq_u64(sum, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); ab = vsha512h2q_u64(intermed, ef, cd); diff --git a/src/hashes/crypto/sha512/dispatch_tables.rs b/src/hashes/crypto/sha512/dispatch_tables.rs index f5bd5fb6..2f95a47f 100644 --- a/src/hashes/crypto/sha512/dispatch_tables.rs +++ b/src/hashes/crypto/sha512/dispatch_tables.rs @@ -3,13 +3,13 @@ //! SHA-512 NI, ARM SHA512 CE, and Zknh have negligible setup cost — use HW //! accel for all size classes when available. -pub use super::kernels::Sha512KernelId as KernelId; +pub(crate) use super::kernels::Sha512KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -17,7 +17,7 @@ pub struct DispatchTable { pub l: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -26,7 +26,7 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "aarch64")] -pub static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Aarch64Sha512, s: KernelId::Aarch64Sha512, @@ -35,7 +35,7 @@ pub static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_SHA512_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_SHA512_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Sha512, s: KernelId::X86Sha512, @@ -44,7 +44,7 @@ pub static X86_SHA512_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx512vl, s: KernelId::X86Avx512vl, @@ -53,7 +53,7 @@ pub static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx2Decoupled, s: KernelId::X86Avx2Decoupled, @@ -62,7 +62,7 @@ pub static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx512vlDecoupled, s: KernelId::X86Avx512vlDecoupled, @@ -71,7 +71,7 @@ pub static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "riscv64")] -pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { +pub(crate) static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Riscv64Zknh, s: KernelId::Riscv64Zknh, @@ -80,7 +80,7 @@ pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "wasm32")] -pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { +pub(crate) static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::WasmSimd128, s: KernelId::WasmSimd128, @@ -89,7 +89,7 @@ pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "s390x")] -pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { +pub(crate) static S390X_KIMD_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::S390xKimd, s: KernelId::S390xKimd, @@ -99,11 +99,11 @@ pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { // x86_64 cascade: SHA-512 NI, then vendor-aware AVX2/AVX-512VL, then portable. // - // The AVX2 kernel falls back to portable for odd block counts, while - // AVX-512VL handles a trailing single block. Vendor-specific ordering is a + // Both AVX2 and AVX-512VL handle a trailing single block inside their SIMD + // kernels. Vendor-specific ordering between the implementations remains a // manually maintained dispatch policy. #[cfg(target_arch = "x86_64")] { @@ -164,5 +164,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D return &S390X_KIMD_TABLE; } } + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/sha512/kernel_test.rs b/src/hashes/crypto/sha512/kernel_test.rs index 1a4dcd2e..63275d98 100644 --- a/src/hashes/crypto/sha512/kernel_test.rs +++ b/src/hashes/crypto/sha512/kernel_test.rs @@ -6,13 +6,6 @@ use super::{ }; use crate::{hashes::crypto::dispatch_util::SizeClassDispatch, traits::Digest as _}; -#[allow(dead_code)] -#[derive(Clone, Debug)] -pub struct KernelResult { - pub name: &'static str, - pub digest: [u8; 64], -} - fn hasher_for_kernel(id: Sha512KernelId) -> Sha512 { let compress = compress_blocks_fn(id); Sha512 { @@ -67,43 +60,16 @@ fn digest_64_byte_prefix_with_kernel(id: Sha512KernelId, prefix: &[u8; 64], data Sha512::digest_64_byte_prefix_with(prefix, data, compress_blocks_fn(id)) } -#[allow(dead_code)] -#[must_use] -pub fn run_all_sha512_kernels(data: &[u8]) -> Vec { - let caps = crate::platform::caps(); - let mut out = Vec::with_capacity(ALL.len()); - for &id in ALL { - if caps.has(required_caps(id)) { - out.push(KernelResult { - name: id.as_str(), - digest: digest_with_kernel(id, data), - }); - } - } - out -} - -#[allow(dead_code)] -pub fn verify_sha512_kernels(data: &[u8]) -> Result<(), &'static str> { - let results = run_all_sha512_kernels(data); - let Some(first) = results.first() else { - return Ok(()); - }; - for r in &results[1..] { - if r.digest != first.digest { - return Err("sha512 kernel mismatch"); - } - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; fn pattern(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(13).wrapping_add((i >> 8) as u8)) + .map(|i| { + let bytes = i.to_le_bytes(); + bytes[0].wrapping_mul(13).wrapping_add(bytes[1]) + }) .collect() } @@ -112,7 +78,7 @@ mod tests { let caps = crate::platform::caps(); #[cfg(not(miri))] let lens = [ - 0usize, 1, 2, 3, 111, 112, 113, 127, 128, 129, 239, 240, 241, 255, 256, 257, 1000, + 0usize, 1, 2, 3, 111, 112, 113, 127, 128, 129, 239, 240, 241, 255, 256, 257, 1000, 4096, ]; #[cfg(miri)] let lens = [0usize, 1, 111, 112, 113, 127, 128, 129, 255, 256, 257]; @@ -195,7 +161,10 @@ mod tests { let mut prefix = [0u8; 64]; for (i, byte) in prefix.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(17).wrapping_add(0x5a); + *byte = u8::try_from(i) + .expect("SHA-512 prefix index fits in u8") + .wrapping_mul(17) + .wrapping_add(0x5a); } for &id in ALL { @@ -222,10 +191,4 @@ mod tests { } } } - - #[test] - fn run_all_agree() { - verify_sha512_kernels(b"abc").expect("kernels should agree"); - verify_sha512_kernels(&pattern(4096)).expect("kernels should agree"); - } } diff --git a/src/hashes/crypto/sha512/kernels.rs b/src/hashes/crypto/sha512/kernels.rs index 2395f469..941f6c8a 100644 --- a/src/hashes/crypto/sha512/kernels.rs +++ b/src/hashes/crypto/sha512/kernels.rs @@ -16,7 +16,7 @@ pub(crate) type CompressBlocksFn = fn(&mut [u64; 8], &[u8]); #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Sha512KernelId { +pub(crate) enum Sha512KernelId { Portable = 0, #[cfg(target_arch = "aarch64")] Aarch64Sha512 = 1, @@ -40,7 +40,7 @@ impl Sha512KernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "aarch64")] @@ -141,7 +141,7 @@ pub(crate) fn compress_blocks_fn(id: Sha512KernelId) -> CompressBlocksFn { #[inline] #[must_use] -pub const fn required_caps(id: Sha512KernelId) -> Caps { +pub(crate) const fn required_caps(id: Sha512KernelId) -> Caps { match id { Sha512KernelId::Portable => Caps::NONE, #[cfg(target_arch = "aarch64")] @@ -165,7 +165,7 @@ pub const fn required_caps(id: Sha512KernelId) -> Caps { // Keep kernel tests focused on backends that runtime dispatch can actually pick. #[cfg(test)] -pub const ALL: &[Sha512KernelId] = &[ +pub(crate) const ALL: &[Sha512KernelId] = &[ Sha512KernelId::Portable, #[cfg(target_arch = "aarch64")] Sha512KernelId::Aarch64Sha512, diff --git a/src/hashes/crypto/sha512/mod.rs b/src/hashes/crypto/sha512/mod.rs index 65459e27..20ff59d4 100644 --- a/src/hashes/crypto/sha512/mod.rs +++ b/src/hashes/crypto/sha512/mod.rs @@ -1,7 +1,5 @@ //! SHA-512 (FIPS 180-4). -#![allow(clippy::indexing_slicing)] // Fixed-size arrays + compression schedule - use self::kernels::CompressBlocksFn; use crate::{ hashes::{ @@ -216,7 +214,7 @@ impl Sha512Prefix { unsafe { core::ptr::write_volatile(word, 0) }; } // SAFETY: bytes_hashed is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } @@ -280,14 +278,11 @@ impl Sha512 { #[inline] pub(crate) fn compress_blocks_portable(state: &mut [u64; 8], blocks: &[u8]) { - debug_assert_eq!(blocks.len() % BLOCK_LEN, 0); - let mut chunks = blocks.chunks_exact(BLOCK_LEN); - for chunk in &mut chunks { - // SAFETY: `chunks_exact(BLOCK_LEN)` yields slices of exactly `BLOCK_LEN` bytes. - let block = unsafe { &*(chunk.as_ptr() as *const [u8; BLOCK_LEN]) }; + let (chunks, remainder) = blocks.as_chunks::(); + for block in chunks { Self::compress_block(state, block); } - debug_assert!(chunks.remainder().is_empty()); + debug_assert!(remainder.is_empty()); } #[inline] @@ -465,7 +460,7 @@ impl Sha512 { #[inline(always)] fn state_to_digest(state: &[u64; 8]) -> [u8; 64] { let mut out = [0u8; 64]; - for (chunk, &word) in out.chunks_exact_mut(8).zip(state.iter()) { + for (chunk, &word) in out.as_chunks_mut::<8>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } out @@ -528,7 +523,6 @@ impl Sha512 { #[cfg(all(feature = "hmac", any(test, feature = "diag")))] #[inline] - #[allow(dead_code)] pub(crate) fn new_with_compress_for_test(compress_blocks: CompressBlocksFn) -> Self { Self { state: H0, @@ -818,9 +812,9 @@ impl Drop for Sha512 { } crate::traits::ct::zeroize(&mut self.block); // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.block_len, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.block_len, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } @@ -871,7 +865,7 @@ mod tests { use core::fmt::Write; let mut s = String::new(); for &b in bytes { - write!(&mut s, "{:02x}", b).unwrap(); + write!(&mut s, "{:02x}", b).expect("writing hexadecimal to String must succeed"); } s } diff --git a/src/hashes/crypto/sha512/riscv64.rs b/src/hashes/crypto/sha512/riscv64.rs index 811d62fa..64a1d277 100644 --- a/src/hashes/crypto/sha512/riscv64.rs +++ b/src/hashes/crypto/sha512/riscv64.rs @@ -8,17 +8,11 @@ //! //! All functions require the `zknh` target feature. -#![allow(clippy::indexing_slicing)] // Fixed-size arrays + compression schedule - #[cfg(target_arch = "riscv64")] use core::arch::riscv64::{sha512sig0, sha512sig1, sha512sum0, sha512sum1}; use super::BLOCK_LEN; -// Safe wrappers for the unsafe Zknh intrinsics. -// SAFETY: only called from within `compress_blocks_zknh` which has -// `#[target_feature(enable = "zknh")]`, guaranteeing the feature is available. - #[inline(always)] fn sum0(x: u64) -> u64 { // SAFETY: `compress_blocks_zknh` calls this wrapper only inside its Zknh target-feature scope. @@ -50,12 +44,9 @@ fn sig1(x: u64) -> u64 { /// Caller must ensure `zknh` CPU feature is available. #[target_feature(enable = "zknh")] pub(crate) unsafe fn compress_blocks_zknh(state: &mut [u64; 8], blocks: &[u8]) { - debug_assert_eq!(blocks.len() % BLOCK_LEN, 0); - let mut chunks = blocks.chunks_exact(BLOCK_LEN); - for chunk in &mut chunks { - // SAFETY: `chunks_exact(BLOCK_LEN)` yields slices of exactly `BLOCK_LEN` bytes. - let block = unsafe { &*(chunk.as_ptr() as *const [u8; BLOCK_LEN]) }; + let (chunks, remainder) = blocks.as_chunks::(); + debug_assert!(remainder.is_empty()); + for block in chunks { super::compress_block_with(state, block, sum0, sum1, sig0, sig1); } - debug_assert!(chunks.remainder().is_empty()); } diff --git a/src/hashes/crypto/sha512/s390x.rs b/src/hashes/crypto/sha512/s390x.rs index 4d50c97f..dbaf1b15 100644 --- a/src/hashes/crypto/sha512/s390x.rs +++ b/src/hashes/crypto/sha512/s390x.rs @@ -13,9 +13,6 @@ //! //! Requires the MSA facility (CPACF). Caller must verify `s390x::MSA`. -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] - use super::BLOCK_LEN; /// SHA-512 block compression via KIMD instruction. diff --git a/src/hashes/crypto/sha512/wasm.rs b/src/hashes/crypto/sha512/wasm.rs index d752c9fb..4f6920ee 100644 --- a/src/hashes/crypto/sha512/wasm.rs +++ b/src/hashes/crypto/sha512/wasm.rs @@ -7,8 +7,6 @@ //! (they depend on W[t-2]/W[t-1] respectively, both from prior rounds), so //! we get true 2-wide parallelism for sigma computations and additions. -#![allow(clippy::indexing_slicing)] // Fixed-size arrays + compression schedule - #[cfg(target_arch = "wasm32")] use core::arch::wasm32::*; @@ -38,12 +36,20 @@ fn small_sigma1_v(x: v128) -> v128 { // Message schedule helpers /// Load 2 big-endian u64 message words from `ptr`, byte-swapping each. +/// +/// # Safety +/// +/// `ptr` must be valid for an unaligned 16-byte read. The caller must execute +/// this function only when WebAssembly SIMD128 is enabled. #[cfg(target_arch = "wasm32")] #[inline(always)] unsafe fn load_be(ptr: *const u8) -> v128 { - // SAFETY: caller guarantees `ptr` is valid for a 16-byte read - // and the simd128 target feature is enabled. - let raw = unsafe { v128_load(ptr as *const v128) }; + // SAFETY: the caller guarantees that `ptr..ptr+16` is readable. The byte + // array has alignment one, and `read_unaligned` accepts any address. + let bytes = unsafe { ptr.cast::<[u8; 16]>().read_unaligned() }; + // SAFETY: `[u8; 16]` and `v128` have the same size, and every SIMD bit + // pattern is valid. + let raw = unsafe { core::mem::transmute::<[u8; 16], v128>(bytes) }; i8x16_shuffle::<7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8>(raw, raw) } @@ -82,6 +88,11 @@ fn schedule_pair(w: &mut [v128; 8], i: usize) { /// /// Message schedule is computed with SIMD (2 u64 words per v128). /// Compression rounds are scalar (sequential dependency chain). +/// +/// # Safety +/// +/// The caller must ensure WebAssembly SIMD128 is available and `blocks` +/// contains only complete SHA-512 blocks. #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] pub(crate) unsafe fn compress_blocks_wasm_simd(state: &mut [u64; 8], blocks: &[u8]) { @@ -136,24 +147,23 @@ pub(crate) unsafe fn compress_blocks_wasm_simd(state: &mut [u64; 8], blocks: &[u } // Rounds 0-15: extract from loaded vectors. - for pair in 0..8 { - let v = wv[pair]; - let w_lo = i64x2_extract_lane::<0>(v) as u64; - let w_hi = i64x2_extract_lane::<1>(v) as u64; + for (pair, &v) in wv.iter().enumerate() { + let w_lo = i64x2_extract_lane::<0>(v).cast_unsigned(); + let w_hi = i64x2_extract_lane::<1>(v).cast_unsigned(); let r = pair.strict_mul(2); sha_round!(K[r], w_lo); - sha_round!(K[r + 1], w_hi); + sha_round!(K[r.strict_add(1)], w_hi); } // Rounds 16-79: expand schedule with SIMD, then extract for scalar rounds. for pair in 8..40 { schedule_pair(&mut wv, pair); let v = wv[pair & 7]; - let w_lo = i64x2_extract_lane::<0>(v) as u64; - let w_hi = i64x2_extract_lane::<1>(v) as u64; + let w_lo = i64x2_extract_lane::<0>(v).cast_unsigned(); + let w_hi = i64x2_extract_lane::<1>(v).cast_unsigned(); let r = pair.strict_mul(2); sha_round!(K[r], w_lo); - sha_round!(K[r + 1], w_hi); + sha_round!(K[r.strict_add(1)], w_hi); } state[0] = state[0].wrapping_add(a); diff --git a/src/hashes/crypto/sha512/x86_64.rs b/src/hashes/crypto/sha512/x86_64.rs index 5d6f422d..74e0d038 100644 --- a/src/hashes/crypto/sha512/x86_64.rs +++ b/src/hashes/crypto/sha512/x86_64.rs @@ -15,9 +15,6 @@ //! All functions require `sha512` and `avx2` target features. //! Callers must verify CPU capabilities before calling. -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] - #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; @@ -160,10 +157,14 @@ static BSWAP64_MASK: [u8; 32] = [ /// /// Given a = [W[n], W[n+1], W[n+2], W[n+3]] and b = [W[n+4], ...], /// returns [W[n+1], W[n+2], W[n+3], W[n+4]]. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available on the executing CPU. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_w_tm7(a: __m256i, b: __m256i) -> __m256i { - // SAFETY: AVX2 intrinsics are available via this function's #[target_feature] attribute. + // SAFETY: The caller guarantees AVX2 support. unsafe { // Swap halves: [a_hi, b_lo] let t = _mm256_permute2x128_si256(a, b, 0x21); diff --git a/src/hashes/crypto/sha512/x86_64_avx2.rs b/src/hashes/crypto/sha512/x86_64_avx2.rs index 993da189..1951db42 100644 --- a/src/hashes/crypto/sha512/x86_64_avx2.rs +++ b/src/hashes/crypto/sha512/x86_64_avx2.rs @@ -22,10 +22,6 @@ //! //! All functions require `avx2` and `bmi2` target features. -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::indexing_slicing)] - #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; @@ -34,6 +30,9 @@ use super::{BLOCK_LEN, K, big_sigma0, big_sigma1, ch, maj}; // 256-bit SIMD sigma (dual-block schedule, 2 × u64 per 128-bit lane) /// σ0(x) = ROTR(1) ^ ROTR(8) ^ SHR(7) — 256-bit, 3-op rotates. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma0_256(x: __m256i) -> __m256i { @@ -46,6 +45,9 @@ unsafe fn small_sigma0_256(x: __m256i) -> __m256i { } /// σ1(x) = ROTR(19) ^ ROTR(61) ^ SHR(6) — 256-bit, 3-op rotates. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma1_256(x: __m256i) -> __m256i { @@ -60,6 +62,9 @@ unsafe fn small_sigma1_256(x: __m256i) -> __m256i { // 128-bit SIMD sigma (single-block schedule, 2 × u64 per register) /// σ0(x) = ROTR(1) ^ ROTR(8) ^ SHR(7) — 128-bit, 3-op rotates. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma0_128(x: __m128i) -> __m128i { @@ -72,6 +77,9 @@ unsafe fn small_sigma0_128(x: __m128i) -> __m128i { } /// σ1(x) = ROTR(19) ^ ROTR(61) ^ SHR(6) — 128-bit, 3-op rotates. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma1_128(x: __m128i) -> __m128i { @@ -90,6 +98,10 @@ unsafe fn small_sigma1_128(x: __m128i) -> __m128i { static BSWAP64_128: [u8; 16] = [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]; /// Load 2 big-endian u64 words from each of two blocks into a __m256i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. Both block +/// pointers must remain valid for 16 readable bytes starting at `offset`. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn load_two_blocks(blk1: *const u8, blk2: *const u8, offset: usize, bswap: __m128i) -> __m256i { @@ -102,33 +114,53 @@ unsafe fn load_two_blocks(blk1: *const u8, blk2: *const u8, offset: usize, bswap } /// Extract two u64 words from the lower 128-bit lane of a __m256i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_lo(v: __m256i) -> (u64, u64) { // SAFETY: SSE intrinsics are available via the caller's #[target_feature] attribute. unsafe { let lo128 = _mm256_castsi256_si128(v); - (_mm_extract_epi64(lo128, 0) as u64, _mm_extract_epi64(lo128, 1) as u64) + ( + _mm_extract_epi64(lo128, 0).cast_unsigned(), + _mm_extract_epi64(lo128, 1).cast_unsigned(), + ) } } /// Extract two u64 words from the upper 128-bit lane of a __m256i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_hi(v: __m256i) -> (u64, u64) { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. unsafe { let hi128 = _mm256_extracti128_si256(v, 1); - (_mm_extract_epi64(hi128, 0) as u64, _mm_extract_epi64(hi128, 1) as u64) + ( + _mm_extract_epi64(hi128, 0).cast_unsigned(), + _mm_extract_epi64(hi128, 1).cast_unsigned(), + ) } } /// Extract two u64 words from a __m128i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_128(v: __m128i) -> (u64, u64) { // SAFETY: SSE intrinsics are available via the caller's #[target_feature] attribute. - unsafe { (_mm_extract_epi64(v, 0) as u64, _mm_extract_epi64(v, 1) as u64) } + unsafe { + ( + _mm_extract_epi64(v, 0).cast_unsigned(), + _mm_extract_epi64(v, 1).cast_unsigned(), + ) + } } // Rotation-based schedule (eliminates ring-buffer index computation) @@ -138,6 +170,9 @@ unsafe fn extract_128(v: __m128i) -> (u64, u64) { /// Same approach as [`schedule_rotate_256`] but for 128-bit registers. /// Eliminates ring-buffer index computation (`wrapping_sub`, `& 7`) in favour /// of fixed-offset array accesses and physical rotation. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_rotate_128(x: &mut [__m128i; 8], k: __m128i) -> __m128i { @@ -173,6 +208,9 @@ unsafe fn schedule_rotate_128(x: &mut [__m128i; 8], k: __m128i) -> __m128i { /// Physically rotates the `x[]` array so that adjacent schedule words are /// always in adjacent registers. This lets `_mm256_alignr_epi8` extract /// cross-register values without `_mm256_permute2x128_si256`. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_rotate_256(x: &mut [__m256i; 8], k: __m256i) -> __m256i { diff --git a/src/hashes/crypto/sha512/x86_64_avx512vl.rs b/src/hashes/crypto/sha512/x86_64_avx512vl.rs index 9d29c714..4062783e 100644 --- a/src/hashes/crypto/sha512/x86_64_avx512vl.rs +++ b/src/hashes/crypto/sha512/x86_64_avx512vl.rs @@ -28,10 +28,6 @@ //! //! All functions require `avx512f`, `avx512vl`, and `bmi2` target features. -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::indexing_slicing)] - #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; @@ -40,6 +36,9 @@ use super::{BLOCK_LEN, K, big_sigma0, big_sigma1, ch, maj}; // 256-bit SIMD sigma with VPRORQ (dual-block schedule) /// σ0(x) = ROTR(1) ^ ROTR(8) ^ SHR(7) — 256-bit, VPRORQ. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma0_256(x: __m256i) -> __m256i { @@ -53,6 +52,9 @@ unsafe fn small_sigma0_256(x: __m256i) -> __m256i { } /// σ1(x) = ROTR(19) ^ ROTR(61) ^ SHR(6) — 256-bit, VPRORQ. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma1_256(x: __m256i) -> __m256i { @@ -68,6 +70,9 @@ unsafe fn small_sigma1_256(x: __m256i) -> __m256i { // 128-bit SIMD sigma with VPRORQ (single-block schedule) /// σ0(x) = ROTR(1) ^ ROTR(8) ^ SHR(7) — 128-bit, VPRORQ. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma0_128(x: __m128i) -> __m128i { @@ -81,6 +86,9 @@ unsafe fn small_sigma0_128(x: __m128i) -> __m128i { } /// σ1(x) = ROTR(19) ^ ROTR(61) ^ SHR(6) — 128-bit, VPRORQ. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma1_128(x: __m128i) -> __m128i { @@ -96,6 +104,9 @@ unsafe fn small_sigma1_128(x: __m128i) -> __m128i { // Dual-block message schedule helpers (256-bit) /// Cross-register extraction (256-bit): [a[1], a[2], a[3], b[0]]. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn cross_lanes_256(a: __m256i, b: __m256i) -> __m256i { @@ -107,6 +118,9 @@ unsafe fn cross_lanes_256(a: __m256i, b: __m256i) -> __m256i { } /// Compute 2 schedule words for both blocks (256-bit ring buffer, VPRORQ). +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_pair_256(w: &mut [__m256i; 8], i: usize) { @@ -127,6 +141,9 @@ unsafe fn schedule_pair_256(w: &mut [__m256i; 8], i: usize) { // Single-block message schedule helpers (128-bit) /// Cross-register extraction (128-bit): [a[1], b[0]]. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn cross_lanes_128(a: __m128i, b: __m128i) -> __m128i { @@ -135,6 +152,9 @@ unsafe fn cross_lanes_128(a: __m128i, b: __m128i) -> __m128i { } /// Compute 2 schedule words for a single block (128-bit ring buffer, VPRORQ). +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_pair_128(w: &mut [__m128i; 8], i: usize) { @@ -159,6 +179,10 @@ unsafe fn schedule_pair_128(w: &mut [__m128i; 8], i: usize) { static BSWAP64_128: [u8; 16] = [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]; /// Load 2 big-endian u64 words from each of two blocks into a __m256i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. Both +/// block pointers must remain valid for 16 readable bytes starting at `offset`. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn load_two_blocks(blk1: *const u8, blk2: *const u8, offset: usize, bswap: __m128i) -> __m256i { @@ -171,33 +195,53 @@ unsafe fn load_two_blocks(blk1: *const u8, blk2: *const u8, offset: usize, bswap } /// Extract two u64 words from the lower 128-bit lane of a __m256i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_lo(v: __m256i) -> (u64, u64) { // SAFETY: SSE intrinsics are available via the caller's #[target_feature] attribute. unsafe { let lo128 = _mm256_castsi256_si128(v); - (_mm_extract_epi64(lo128, 0) as u64, _mm_extract_epi64(lo128, 1) as u64) + ( + _mm_extract_epi64(lo128, 0).cast_unsigned(), + _mm_extract_epi64(lo128, 1).cast_unsigned(), + ) } } /// Extract two u64 words from the upper 128-bit lane of a __m256i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_hi(v: __m256i) -> (u64, u64) { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. unsafe { let hi128 = _mm256_extracti128_si256(v, 1); - (_mm_extract_epi64(hi128, 0) as u64, _mm_extract_epi64(hi128, 1) as u64) + ( + _mm_extract_epi64(hi128, 0).cast_unsigned(), + _mm_extract_epi64(hi128, 1).cast_unsigned(), + ) } } /// Extract two u64 words from a __m128i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_128(v: __m128i) -> (u64, u64) { // SAFETY: SSE intrinsics are available via the caller's #[target_feature] attribute. - unsafe { (_mm_extract_epi64(v, 0) as u64, _mm_extract_epi64(v, 1) as u64) } + unsafe { + ( + _mm_extract_epi64(v, 0).cast_unsigned(), + _mm_extract_epi64(v, 1).cast_unsigned(), + ) + } } // Entry point @@ -418,6 +462,9 @@ pub(crate) unsafe fn compress_blocks_avx512vl(state: &mut [u64; 8], blocks: &[u8 /// /// Same approach as [`schedule_rotate_256_vl`] but for 128-bit registers. /// Uses VPRORQ native rotates for sigma functions. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_rotate_128(x: &mut [__m128i; 8], k: __m128i) -> __m128i { @@ -454,6 +501,9 @@ unsafe fn schedule_rotate_128(x: &mut [__m128i; 8], k: __m128i) -> __m128i { /// shift-shift-or of the AVX2 path. Combined with array rotation, this also /// eliminates `_mm256_permute2x128_si256`, which the ring-buffer schedule uses /// for cross-lane extraction. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_rotate_256(x: &mut [__m256i; 8], k: __m256i) -> __m256i { diff --git a/src/hashes/crypto/sha512_256.rs b/src/hashes/crypto/sha512_256.rs index baa1d8e9..11f3ffb1 100644 --- a/src/hashes/crypto/sha512_256.rs +++ b/src/hashes/crypto/sha512_256.rs @@ -3,8 +3,6 @@ //! SHA-512/256 is identical to SHA-512 except for initial hash values (H0) and //! output truncation (32 bytes / 4 words). The compression function is shared. -#![allow(clippy::indexing_slicing)] // Fixed-size arrays in finalization - use self::kernels::CompressBlocksFn; use super::sha512::Sha512; use crate::{ @@ -153,7 +151,7 @@ impl Sha512_256 { let state = Sha512::finalize_state_words(self.state, self.block, self.block_len, total_len, compress_blocks); let mut out = [0u8; 32]; - for (chunk, &word) in out.chunks_exact_mut(8).zip(state.iter()) { + for (chunk, &word) in out.as_chunks_mut::<8>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } out @@ -168,9 +166,9 @@ impl Drop for Sha512_256 { } crate::traits::ct::zeroize(&mut self.block); // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.block_len, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.block_len, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } diff --git a/src/hashes/crypto/sha512_256/dispatch_tables.rs b/src/hashes/crypto/sha512_256/dispatch_tables.rs index 0b5c5586..0d391f11 100644 --- a/src/hashes/crypto/sha512_256/dispatch_tables.rs +++ b/src/hashes/crypto/sha512_256/dispatch_tables.rs @@ -3,13 +3,13 @@ //! SHA-512/256 uses identical compression to SHA-512, so the same hardware kernels //! and cascade order apply. -pub use super::kernels::Sha512_256KernelId as KernelId; +pub(crate) use super::kernels::Sha512_256KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -17,7 +17,7 @@ pub struct DispatchTable { pub l: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -26,7 +26,7 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "aarch64")] -pub static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Aarch64Sha512, s: KernelId::Aarch64Sha512, @@ -35,7 +35,7 @@ pub static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_SHA512_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_SHA512_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Sha512, s: KernelId::X86Sha512, @@ -44,7 +44,7 @@ pub static X86_SHA512_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx512vl, s: KernelId::X86Avx512vl, @@ -53,7 +53,7 @@ pub static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx2Decoupled, s: KernelId::X86Avx2Decoupled, @@ -62,7 +62,7 @@ pub static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx512vlDecoupled, s: KernelId::X86Avx512vlDecoupled, @@ -71,7 +71,7 @@ pub static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "riscv64")] -pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { +pub(crate) static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Riscv64Zknh, s: KernelId::Riscv64Zknh, @@ -80,7 +80,7 @@ pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "wasm32")] -pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { +pub(crate) static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::WasmSimd128, s: KernelId::WasmSimd128, @@ -89,7 +89,7 @@ pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "s390x")] -pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { +pub(crate) static S390X_KIMD_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::S390xKimd, s: KernelId::S390xKimd, @@ -99,7 +99,7 @@ pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { // x86_64 cascade: SHA-512 NI > vendor-aware AVX2/AVX-512VL > Portable // AMD: AVX2 decoupled > AVX-512VL; Intel: AVX-512VL decoupled > AVX2 decoupled. // See sha512/dispatch_tables.rs for full rationale. @@ -157,5 +157,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D return &S390X_KIMD_TABLE; } } + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/sha512_256/kernel_test.rs b/src/hashes/crypto/sha512_256/kernel_test.rs index d86b3024..1d13ac90 100644 --- a/src/hashes/crypto/sha512_256/kernel_test.rs +++ b/src/hashes/crypto/sha512_256/kernel_test.rs @@ -6,13 +6,6 @@ use super::{ }; use crate::{hashes::crypto::dispatch_util::SizeClassDispatch, traits::Digest as _}; -#[allow(dead_code)] -#[derive(Clone, Debug)] -pub struct KernelResult { - pub name: &'static str, - pub digest: [u8; 32], -} - fn hasher_for_kernel(id: Sha512_256KernelId) -> Sha512_256 { let compress = compress_blocks_fn(id); Sha512_256 { @@ -63,43 +56,16 @@ fn digest_oneshot_with_kernel(id: Sha512_256KernelId, data: &[u8]) -> [u8; 32] { out } -#[allow(dead_code)] -#[must_use] -pub fn run_all_sha512_256_kernels(data: &[u8]) -> Vec { - let caps = crate::platform::caps(); - let mut out = Vec::with_capacity(ALL.len()); - for &id in ALL { - if caps.has(required_caps(id)) { - out.push(KernelResult { - name: id.as_str(), - digest: digest_with_kernel(id, data), - }); - } - } - out -} - -#[allow(dead_code)] -pub fn verify_sha512_256_kernels(data: &[u8]) -> Result<(), &'static str> { - let results = run_all_sha512_256_kernels(data); - let Some(first) = results.first() else { - return Ok(()); - }; - for r in &results[1..] { - if r.digest != first.digest { - return Err("sha512-256 kernel mismatch"); - } - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; fn pattern(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(29).wrapping_add((i >> 8) as u8)) + .map(|i| { + let bytes = i.to_le_bytes(); + bytes[0].wrapping_mul(29).wrapping_add(bytes[1]) + }) .collect() } @@ -108,7 +74,7 @@ mod tests { let caps = crate::platform::caps(); #[cfg(not(miri))] let lens = [ - 0usize, 1, 2, 3, 111, 112, 113, 127, 128, 129, 239, 240, 241, 255, 256, 257, 1000, + 0usize, 1, 2, 3, 111, 112, 113, 127, 128, 129, 239, 240, 241, 255, 256, 257, 1000, 4096, ]; #[cfg(miri)] let lens = [0usize, 1, 111, 112, 113, 127, 128, 129, 255, 256, 257]; diff --git a/src/hashes/crypto/sha512_256/kernels.rs b/src/hashes/crypto/sha512_256/kernels.rs index 869c0661..e92032e7 100644 --- a/src/hashes/crypto/sha512_256/kernels.rs +++ b/src/hashes/crypto/sha512_256/kernels.rs @@ -6,7 +6,7 @@ use crate::{hashes::crypto::sha512::Sha512, platform::Caps}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Sha512_256KernelId { +pub(crate) enum Sha512_256KernelId { Portable = 0, #[cfg(target_arch = "aarch64")] Aarch64Sha512 = 1, @@ -30,7 +30,7 @@ impl Sha512_256KernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "aarch64")] @@ -120,13 +120,13 @@ pub(crate) fn compress_blocks_fn(id: Sha512_256KernelId) -> CompressBlocksFn { #[inline] #[must_use] -pub const fn required_caps(id: Sha512_256KernelId) -> Caps { +pub(crate) const fn required_caps(id: Sha512_256KernelId) -> Caps { crate::hashes::crypto::sha512::kernels::required_caps(to_sha512_kernel_id(id)) } // Keep kernel tests focused on backends that runtime dispatch can actually pick. #[cfg(test)] -pub const ALL: &[Sha512_256KernelId] = &[ +pub(crate) const ALL: &[Sha512_256KernelId] = &[ Sha512_256KernelId::Portable, #[cfg(target_arch = "aarch64")] Sha512_256KernelId::Aarch64Sha512, diff --git a/src/hashes/crypto/sp800185.rs b/src/hashes/crypto/sp800185.rs index 0e45cbaf..53b2be62 100644 --- a/src/hashes/crypto/sp800185.rs +++ b/src/hashes/crypto/sp800185.rs @@ -1,7 +1,5 @@ //! SP 800-185 framing helpers shared by cSHAKE and KMAC. -#![allow(clippy::indexing_slicing)] // Fixed-width 9-byte encodings and rate-sized padding buffers. - use super::keccak::KeccakCore; /// cSHAKE128 / KMAC128 bitrate in bytes. @@ -18,10 +16,10 @@ fn encode_u64_be(value: u64, out: &mut [u8; 9], right: bool) -> usize { if right { out[..width].copy_from_slice(&bytes[first..]); - out[width] = width as u8; + out[width] = u8::try_from(width).expect("SP 800-185 integer width is at most eight bytes"); width.strict_add(1) } else { - out[0] = width as u8; + out[0] = u8::try_from(width).expect("SP 800-185 integer width is at most eight bytes"); out[1..=width].copy_from_slice(&bytes[first..]); width.strict_add(1) } @@ -48,14 +46,15 @@ pub(crate) fn encoded_string_len(data: &[u8]) -> usize { } pub(crate) fn absorb_bytepad(core: &mut KeccakCore, segments: &[&[u8]], payload_len: usize) { - let (prefix, prefix_len) = left_encode(RATE as u64); + let rate = u64::try_from(RATE).expect("Keccak rate must fit u64"); + let (prefix, prefix_len) = left_encode(rate); core.update(&prefix[..prefix_len]); for segment in segments { core.update(segment); } let total_len = prefix_len.strict_add(payload_len); - let rem = total_len % RATE; + let rem = total_len.strict_rem(RATE); if rem != 0 { core.update(&[0u8; RATE][..RATE.strict_sub(rem)]); } diff --git a/src/hashes/fast/rapidhash.rs b/src/hashes/fast/rapidhash.rs index dad39ee1..d83b016f 100644 --- a/src/hashes/fast/rapidhash.rs +++ b/src/hashes/fast/rapidhash.rs @@ -2,8 +2,6 @@ //! //! The portable, safe Rust implementation is the sole semantic authority. -#![allow(clippy::indexing_slicing)] - use crate::traits::FastHash; mod stream; @@ -89,32 +87,49 @@ impl FastHash for RapidHash64 { #[inline(always)] const fn read_u32_le(input: &[u8], offset: usize) -> u32 { - let (_, tail) = input.split_at(offset); + let Some((_, tail)) = input.split_at_checked(offset) else { + return 0; + }; let Some(bytes) = tail.first_chunk::<4>() else { - panic!("RapidHash u32 read exceeds input"); + return 0; }; u32::from_le_bytes(*bytes) } #[inline(always)] const fn read_u64_le(input: &[u8], offset: usize) -> u64 { - let (_, tail) = input.split_at(offset); + let Some((_, tail)) = input.split_at_checked(offset) else { + return 0; + }; let Some(bytes) = tail.first_chunk::<8>() else { - panic!("RapidHash u64 read exceeds input"); + return 0; }; u64::from_le_bytes(*bytes) } +#[inline(always)] +const fn u128_words(value: u128) -> (u64, u64) { + let bytes = value.to_le_bytes(); + let low = u64::from_le_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ]); + let high = u64::from_le_bytes([ + bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15], + ]); + (low, high) +} + #[inline(always)] const fn rapid_mum(a: u64, b: u64) -> (u64, u64) { let product = (a as u128).wrapping_mul(b as u128); - (product as u64, (product >> 64) as u64) + u128_words(product) } #[inline(always)] const fn rapid_mix(a: u64, b: u64) -> u64 { let product = (a as u128).wrapping_mul(b as u128); - (product as u64) ^ ((product >> 64) as u64) + let (low, high) = u128_words(product); + low ^ high } #[inline(always)] @@ -192,12 +207,7 @@ const fn rapidhash_core_large(data: &[u8], mut seed: u64, secrets: &[u64; 7]) -> // Validate the span once so the fixed-size safe reads need no per-load bounds checks. let (_, remaining) = data.split_at(offset); let (block, _) = remaining.split_at(224); - let Some(first) = block.first_chunk::<112>() else { - panic!("RapidHash block is shorter than 112 bytes"); - }; - let Some(second) = block.last_chunk::<112>() else { - panic!("RapidHash block is shorter than 224 bytes"); - }; + let (first, second) = block.split_at(112); seed = rapid_mix(read_u64_le(first, 0) ^ secrets[0], read_u64_le(first, 8) ^ seed); see1 = rapid_mix(read_u64_le(first, 16) ^ secrets[1], read_u64_le(first, 24) ^ see1); see2 = rapid_mix(read_u64_le(first, 32) ^ secrets[2], read_u64_le(first, 40) ^ see2); @@ -218,9 +228,7 @@ const fn rapidhash_core_large(data: &[u8], mut seed: u64, secrets: &[u64; 7]) -> if data.len().strict_sub(offset) > 112 { let (_, remaining) = data.split_at(offset); - let Some(block) = remaining.first_chunk::<112>() else { - panic!("RapidHash block is shorter than 112 bytes"); - }; + let (block, _) = remaining.split_at(112); seed = rapid_mix(read_u64_le(block, 0) ^ secrets[0], read_u64_le(block, 8) ^ seed); see1 = rapid_mix(read_u64_le(block, 16) ^ secrets[1], read_u64_le(block, 24) ^ see1); see2 = rapid_mix(read_u64_le(block, 32) ^ secrets[2], read_u64_le(block, 40) ^ see2); @@ -303,7 +311,9 @@ mod tests { const CONST_SEEDED: u64 = RapidHash64::hash_with_seed(42, b"const RapidHash"); fn data(len: usize) -> Vec { - (0..len).map(|i| i.wrapping_mul(131).wrapping_add(17) as u8).collect() + (0..len) + .map(|i| i.wrapping_mul(131).wrapping_add(17).to_le_bytes()[0]) + .collect() } fn reference(seed: u64, data: &[u8]) -> u64 { diff --git a/src/hashes/fast/rapidhash/stream.rs b/src/hashes/fast/rapidhash/stream.rs index 568ea334..079dcc7a 100644 --- a/src/hashes/fast/rapidhash/stream.rs +++ b/src/hashes/fast/rapidhash/stream.rs @@ -2,7 +2,7 @@ use core::hash::{BuildHasher, Hasher}; use super::{ DEFAULT_SECRETS, RapidSecrets, premix_seed, rapid_mix, rapid_mum, rapidhash_core, rapidhash_seed_cpp, read_u32_le, - read_u64_le, + read_u64_le, u128_words, }; const CHUNK_SIZE: usize = 112; @@ -76,7 +76,11 @@ impl RapidStreamHasher { &data[copy_len..] }; - let stop = remaining.len().saturating_sub(1) / CHUNK_SIZE * CHUNK_SIZE; + let stop = remaining + .len() + .saturating_sub(1) + .strict_div(CHUNK_SIZE) + .strict_mul(CHUNK_SIZE); let mut last_chunk = None; let mut offset = 0usize; while offset < stop { @@ -242,10 +246,8 @@ impl RapidHasher { #[inline(always)] fn flush_sponge(&mut self) { if self.sponge_bits != 0 { - self.seed = rapid_mix( - self.sponge as u64 ^ self.seed, - (self.sponge >> 64) as u64 ^ self.word0(), - ); + let (low, high) = u128_words(self.sponge); + self.seed = rapid_mix(low ^ self.seed, high ^ self.word0()); self.sponge = 0; self.sponge_bits = 0; } @@ -256,10 +258,8 @@ impl RapidHasher { if self.sponge_bits == 0 { self.seed } else { - rapid_mix( - self.sponge as u64 ^ self.seed, - (self.sponge >> 64) as u64 ^ self.word0(), - ) + let (low, high) = u128_words(self.sponge); + rapid_mix(low ^ self.seed, high ^ self.word0()) } } } @@ -284,20 +284,26 @@ fn hash_deterministic(bytes: &[u8], seed: u64) -> u64 { } macro_rules! write_integer { - ($($method:ident, $ty:ty, $unsigned:ty),+ $(,)?) => { + (@convert unsigned, $value:expr) => { + $value as u128 + }; + (@convert signed, $value:expr) => { + $value.cast_unsigned() as u128 + }; + ($kind:ident; $($method:ident, $ty:ty),+ $(,)?) => { $( #[inline(always)] fn $method(&mut self, value: $ty) { - const BITS: u8 = core::mem::size_of::<$ty>() as u8 * 8; - let value = (value as $unsigned) as u128; - let next_bits = self.sponge_bits.strict_add(BITS); + let bits = u8::try_from(<$ty>::BITS).expect("Rust integer width should fit in u8"); + let value = write_integer!(@convert $kind, value); + let next_bits = self.sponge_bits.strict_add(bits); if next_bits <= 128 { self.sponge |= value << self.sponge_bits; self.sponge_bits = next_bits; } else { self.flush_sponge(); self.sponge = value; - self.sponge_bits = BITS; + self.sponge_bits = bits; } } )+ @@ -324,36 +330,20 @@ impl Hasher for RapidHasher { } write_integer!( - write_u8, - u8, - u8, - write_u16, - u16, - u16, - write_u32, - u32, - u32, - write_u64, - u64, - u64, - write_usize, - usize, - usize, - write_i8, - i8, - u8, - write_i16, - i16, - u16, - write_i32, - i32, - u32, - write_i64, - i64, - u64, - write_isize, - isize, - usize, + unsigned; + write_u8, u8, + write_u16, u16, + write_u32, u32, + write_u64, u64, + write_usize, usize, + ); + write_integer!( + signed; + write_i8, i8, + write_i16, i16, + write_i32, i32, + write_i64, i64, + write_isize, isize, ); #[inline(always)] @@ -365,7 +355,7 @@ impl Hasher for RapidHasher { #[inline(always)] fn write_i128(&mut self, value: i128) { - self.write_u128(value as u128); + self.write_u128(value.cast_unsigned()); } } @@ -452,7 +442,9 @@ mod tests { use super::*; fn data(len: usize) -> Vec { - (0..len).map(|i| i.wrapping_mul(131).wrapping_add(17) as u8).collect() + (0..len) + .map(|i| i.wrapping_mul(131).wrapping_add(17).to_le_bytes()[0]) + .collect() } fn seeded_hasher(seed: u64) -> RapidHasher { @@ -571,21 +563,22 @@ mod tests { seed.copy_from_slice(&1u64.to_le_bytes()); Ok::<_, ()>(()) }) - .unwrap(); + .expect("caller-provided entropy should initialize the first state"); let same = RapidRandomState::try_new_with(|seed| { seed.copy_from_slice(&1u64.to_le_bytes()); Ok::<_, ()>(()) }) - .unwrap(); + .expect("caller-provided entropy should initialize the matching state"); let second = RapidRandomState::try_new_with(|seed| { seed.copy_from_slice(&2u64.to_le_bytes()); Ok::<_, ()>(()) }) - .unwrap(); + .expect("caller-provided entropy should initialize the second state"); assert_eq!(first.hash_one(b"collection key"), same.hash_one(b"collection key")); assert_ne!(first.hash_one(b"collection key"), second.hash_one(b"collection key")); - assert!(RapidRandomState::try_new_with(|_| Err::<(), _>("entropy unavailable")).is_err()); + RapidRandomState::try_new_with(|_| Err::<(), _>("entropy unavailable")) + .expect_err("entropy-source failures should be returned"); } #[test] diff --git a/src/hashes/fast/xxh3.rs b/src/hashes/fast/xxh3.rs index 402990d3..8aaeec53 100644 --- a/src/hashes/fast/xxh3.rs +++ b/src/hashes/fast/xxh3.rs @@ -3,8 +3,6 @@ //! Hardware-accelerated on x86-64 (AVX2, AVX-512), aarch64 (NEON), POWER //! (VSX), and s390x (z/Vector), with a portable scalar fallback. -#![allow(clippy::indexing_slicing)] // Tight block parsing + fixed-size arrays - use core::mem; use crate::traits::FastHash; @@ -30,9 +28,11 @@ pub(crate) mod x86_64_avx2; pub(crate) mod x86_64_avx512; #[derive(Clone, Debug, Default)] +/// XXH3 64-bit non-cryptographic hash. pub struct Xxh3_64; #[derive(Clone, Debug, Default)] +/// XXH3 128-bit non-cryptographic hash. pub struct Xxh3_128; // xxh32 primes (used in scramble/mix) @@ -92,10 +92,10 @@ const INITIAL_ACC: [u64; ACC_NB] = [ /// Caller must ensure `offset + 4 <= input.len()`. #[inline(always)] unsafe fn read_u32_le(input: &[u8], offset: usize) -> u32 { - debug_assert!(offset + 4 <= input.len()); + debug_assert!(offset.strict_add(4) <= input.len()); // SAFETY: caller ensures `offset + 4 <= input.len()`, and `read_unaligned` supports unaligned // loads. - let v = unsafe { core::ptr::read_unaligned(input.as_ptr().add(offset) as *const u32) }; + let v = unsafe { core::ptr::read_unaligned(input.as_ptr().add(offset).cast::()) }; u32::from_le(v) } @@ -104,10 +104,10 @@ unsafe fn read_u32_le(input: &[u8], offset: usize) -> u32 { /// Caller must ensure `offset + 8 <= input.len()`. #[inline(always)] unsafe fn read_u64_le(input: &[u8], offset: usize) -> u64 { - debug_assert!(offset + 8 <= input.len()); + debug_assert!(offset.strict_add(8) <= input.len()); // SAFETY: caller ensures `offset + 8 <= input.len()`, and `read_unaligned` supports unaligned // loads. - let v = unsafe { core::ptr::read_unaligned(input.as_ptr().add(offset) as *const u64) }; + let v = unsafe { core::ptr::read_unaligned(input.as_ptr().add(offset).cast::()) }; u64::from_le(v) } @@ -120,9 +120,9 @@ unsafe fn read_u64_le(input: &[u8], offset: usize) -> u64 { /// Caller must ensure `offset + 16 <= data.len()`. #[inline(always)] unsafe fn chunk16(data: &[u8], offset: usize) -> &[[u8; 8]; 2] { - debug_assert!(offset + 16 <= data.len()); + debug_assert!(offset.strict_add(16) <= data.len()); // SAFETY: caller ensures bounds. The resulting reference is valid for 16 bytes. - unsafe { &*(data.as_ptr().add(offset) as *const [[u8; 8]; 2]) } + unsafe { &*data.as_ptr().add(offset).cast::<[[u8; 8]; 2]>() } } /// Extract two consecutive 16-byte chunks (32 bytes) at `offset`. @@ -135,14 +135,29 @@ unsafe fn chunk16(data: &[u8], offset: usize) -> &[[u8; 8]; 2] { /// Caller must ensure `offset + 32 <= data.len()`. #[inline(always)] unsafe fn chunk32(data: &[u8], offset: usize) -> &[[[u8; 8]; 2]; 2] { - debug_assert!(offset + 32 <= data.len()); + debug_assert!(offset.strict_add(32) <= data.len()); // SAFETY: caller ensures bounds. The resulting reference is valid for 32 bytes. - unsafe { &*(data.as_ptr().add(offset) as *const [[[u8; 8]; 2]; 2]) } + unsafe { &*data.as_ptr().add(offset).cast::<[[[u8; 8]; 2]; 2]>() } } #[inline(always)] const fn mult32_to64(left: u32, right: u32) -> u64 { - (left as u64).wrapping_mul(right as u64) + (left as u64).strict_mul(right as u64) +} + +#[inline(always)] +const fn low_u32(value: u64) -> u32 { + let [b0, b1, b2, b3, ..] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) +} + +#[inline(always)] +const fn split_u128(value: u128) -> (u64, u64) { + let [b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15] = value.to_le_bytes(); + ( + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]), + u64::from_le_bytes([b8, b9, b10, b11, b12, b13, b14, b15]), + ) } #[inline(always)] @@ -168,8 +183,7 @@ const fn strong_avalanche(mut value: u64, len: u64) -> u64 { #[inline(always)] const fn mul64_to128(left: u64, right: u64) -> (u64, u64) { - let product = (left as u128).wrapping_mul(right as u128); - (product as u64, (product >> 64) as u64) + split_u128((left as u128).strict_mul(right as u128)) } #[inline(always)] @@ -229,7 +243,7 @@ fn xxh3_64_9to16(input: &[u8], seed: u64, secret: &[u8]) -> u64 { let flip2 = (read_u64_le(secret, 40) ^ read_u64_le(secret, 48)).wrapping_sub(seed); let input_lo = read_u64_le(input, 0) ^ flip1; - let input_hi = read_u64_le(input, input.len() - 8) ^ flip2; + let input_hi = read_u64_le(input, input.len().strict_sub(8)) ^ flip2; let acc = (input.len() as u64) .wrapping_add(input_lo.swap_bytes()) @@ -242,12 +256,12 @@ fn xxh3_64_9to16(input: &[u8], seed: u64, secret: &[u8]) -> u64 { #[inline(always)] fn xxh3_64_4to8(input: &[u8], mut seed: u64, secret: &[u8]) -> u64 { - seed ^= ((seed as u32).swap_bytes() as u64) << 32; + seed ^= u64::from(low_u32(seed).swap_bytes()) << 32; // SAFETY: input.len() is 4..=8, secret.len() >= SECRET_SIZE_MIN (136). unsafe { let input1 = read_u32_le(input, 0); - let input2 = read_u32_le(input, input.len() - 4); + let input2 = read_u32_le(input, input.len().strict_sub(4)); let flip = (read_u64_le(secret, 8) ^ read_u64_le(secret, 16)).wrapping_sub(seed); let input64 = (input2 as u64).wrapping_add((input1 as u64) << 32); @@ -264,7 +278,8 @@ fn xxh3_64_1to3(input: &[u8], seed: u64, secret: &[u8]) -> u64 { let c1 = *input.get_unchecked(0); let c2 = *input.get_unchecked(input.len() >> 1); let c3 = *input.get_unchecked(input.len().strict_sub(1)); - let combo = ((c1 as u32) << 16) | ((c2 as u32) << 24) | (c3 as u32) | ((input.len() as u32) << 8); + let input_len = u32::try_from(input.len()).expect("XXH3 1-to-3-byte path length must fit u32"); + let combo = (u32::from(c1) << 16) | (u32::from(c2) << 24) | u32::from(c3) | (input_len << 8); let flip = ((read_u32_le(secret, 0) ^ read_u32_le(secret, 4)) as u64).wrapping_add(seed); xxh64_avalanche((combo as u64) ^ flip) @@ -296,19 +311,35 @@ fn xxh3_64_7to128(input: &[u8], seed: u64, secret: &[u8]) -> u64 { if input.len() > 64 { if input.len() > 96 { acc = acc.wrapping_add(mix16_b(chunk16(input, 48), chunk16(secret, 96), seed)); - acc = acc.wrapping_add(mix16_b(chunk16(input, input.len() - 64), chunk16(secret, 112), seed)); + acc = acc.wrapping_add(mix16_b( + chunk16(input, input.len().strict_sub(64)), + chunk16(secret, 112), + seed, + )); } acc = acc.wrapping_add(mix16_b(chunk16(input, 32), chunk16(secret, 64), seed)); - acc = acc.wrapping_add(mix16_b(chunk16(input, input.len() - 48), chunk16(secret, 80), seed)); + acc = acc.wrapping_add(mix16_b( + chunk16(input, input.len().strict_sub(48)), + chunk16(secret, 80), + seed, + )); } acc = acc.wrapping_add(mix16_b(chunk16(input, 16), chunk16(secret, 32), seed)); - acc = acc.wrapping_add(mix16_b(chunk16(input, input.len() - 32), chunk16(secret, 48), seed)); + acc = acc.wrapping_add(mix16_b( + chunk16(input, input.len().strict_sub(32)), + chunk16(secret, 48), + seed, + )); } acc = acc.wrapping_add(mix16_b(chunk16(input, 0), chunk16(secret, 0), seed)); - acc = acc.wrapping_add(mix16_b(chunk16(input, input.len() - 16), chunk16(secret, 16), seed)); + acc = acc.wrapping_add(mix16_b( + chunk16(input, input.len().strict_sub(16)), + chunk16(secret, 16), + seed, + )); xxh3_avalanche(acc) } @@ -361,22 +392,25 @@ fn xxh3_64_129to240(input: &[u8], seed: u64, secret: &[u8]) -> u64 { let mut idx = 0usize; while idx < 8 { - acc = acc.wrapping_add(mix16_b(chunk16(input, 16 * idx), chunk16(secret, 16 * idx), seed)); - idx += 1; + let offset = idx.strict_mul(16); + acc = acc.wrapping_add(mix16_b(chunk16(input, offset), chunk16(secret, offset), seed)); + idx = idx.strict_add(1); } acc = xxh3_avalanche(acc); while idx < nb_rounds { + let input_offset = idx.strict_mul(16); + let secret_offset = idx.strict_sub(8).strict_mul(16).strict_add(START_OFFSET); acc = acc.wrapping_add(mix16_b( - chunk16(input, 16 * idx), - chunk16(secret, 16 * (idx - 8) + START_OFFSET), + chunk16(input, input_offset), + chunk16(secret, secret_offset), seed, )); - idx += 1; + idx = idx.strict_add(1); } acc = acc.wrapping_add(mix16_b( - chunk16(input, input.len() - 16), + chunk16(input, input.len().strict_sub(16)), chunk16(secret, SECRET_SIZE_MIN - LAST_OFFSET), seed, )); @@ -391,7 +425,7 @@ fn mix_two_accs(acc: &[u64], acc_offset: usize, secret: &[u8], secret_offset: us unsafe { mul128_fold64( acc[acc_offset] ^ read_u64_le(secret, secret_offset), - acc[acc_offset + 1] ^ read_u64_le(secret, secret_offset + 8), + acc[acc_offset.strict_add(1)] ^ read_u64_le(secret, secret_offset.strict_add(8)), ) } } @@ -400,8 +434,10 @@ fn mix_two_accs(acc: &[u64], acc_offset: usize, secret: &[u8], secret_offset: us fn merge_accs(acc: &[u64], secret: &[u8], secret_offset: usize, mut result: u64) -> u64 { let mut idx = 0usize; while idx < 4 { - result = result.wrapping_add(mix_two_accs(acc, idx * 2, secret, secret_offset + idx * 16)); - idx += 1; + let acc_offset = idx.strict_mul(2); + let key_offset = secret_offset.strict_add(idx.strict_mul(16)); + result = result.wrapping_add(mix_two_accs(acc, acc_offset, secret, key_offset)); + idx = idx.strict_add(1); } xxh3_avalanche(result) @@ -409,7 +445,8 @@ fn merge_accs(acc: &[u64], secret: &[u8], secret_offset: usize, mut result: u64) #[inline(always)] fn scramble_acc(mut acc: [u64; ACC_NB], secret: &[u8], secret_offset: usize) -> [u64; ACC_NB] { - let secret_stripe = &secret[secret_offset..secret_offset + STRIPE_LEN]; + let secret_end = secret_offset.strict_add(STRIPE_LEN); + let secret_stripe = &secret[secret_offset..secret_end]; let (secret_chunks, _) = secret_stripe.as_chunks::<8>(); let mut idx = 0usize; @@ -419,7 +456,7 @@ fn scramble_acc(mut acc: [u64; ACC_NB], secret: &[u8], secret_offset: usize) -> acc_val ^= key; acc[idx] = acc_val.wrapping_mul(PRIME32_1 as u64); - idx += 1; + idx = idx.strict_add(1); } acc @@ -440,7 +477,7 @@ fn accumulate_512(mut acc: [u64; ACC_NB], stripe: &[u8], secret_stripe: &[u8]) - acc[idx ^ 1] = acc[idx ^ 1].wrapping_add(data_val); acc[idx] = acc[idx].wrapping_add(mult32_to64((data_key & 0xFFFF_FFFF) as u32, (data_key >> 32) as u32)); - idx += 1; + idx = idx.strict_add(1); } acc @@ -462,16 +499,18 @@ fn accumulate_loop( let mut idx = 0usize; while idx < nb_stripes { - debug_assert!(stripe_offset + STRIPE_LEN <= input_len); - debug_assert!(secret_stripe_offset + STRIPE_LEN <= secret_len); + let stripe_end = stripe_offset.strict_add(STRIPE_LEN); + let secret_stripe_end = secret_stripe_offset.strict_add(STRIPE_LEN); + debug_assert!(stripe_end <= input_len); + debug_assert!(secret_stripe_end <= secret_len); acc = accumulate_512( acc, - &input[stripe_offset..stripe_offset + STRIPE_LEN], - &secret[secret_stripe_offset..secret_stripe_offset + STRIPE_LEN], + &input[stripe_offset..stripe_end], + &secret[secret_stripe_offset..secret_stripe_end], ); - stripe_offset += STRIPE_LEN; - secret_stripe_offset += SECRET_CONSUME_RATE; - idx += 1; + stripe_offset = stripe_end; + secret_stripe_offset = secret_stripe_offset.strict_add(SECRET_CONSUME_RATE); + idx = idx.strict_add(1); } acc @@ -496,31 +535,48 @@ fn stream_accumulate_portable( } #[inline(never)] +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { let mut acc = INITIAL_ACC; - let nb_stripes = (secret.len() - STRIPE_LEN) / SECRET_CONSUME_RATE; - let block_len = STRIPE_LEN * nb_stripes; - let nb_blocks = (input.len() - 1) / block_len; + let nb_stripes = secret.len().strict_sub(STRIPE_LEN) / SECRET_CONSUME_RATE; + let block_len = STRIPE_LEN.strict_mul(nb_stripes); + let remaining = input.len().strict_sub(1); + let nb_blocks = remaining.strict_div(block_len); let mut idx = 0usize; while idx < nb_blocks { - acc = accumulate_loop(acc, input, idx * block_len, secret, 0, nb_stripes); - acc = scramble_acc(acc, secret, secret.len() - STRIPE_LEN); - idx += 1; + let block_offset = idx.strict_mul(block_len); + acc = accumulate_loop(acc, input, block_offset, secret, 0, nb_stripes); + acc = scramble_acc(acc, secret, secret.len().strict_sub(STRIPE_LEN)); + idx = idx.strict_add(1); } - let nb_stripes = ((input.len() - 1) - (block_len * nb_blocks)) / STRIPE_LEN; - acc = accumulate_loop(acc, input, nb_blocks * block_len, secret, 0, nb_stripes); + let block_offset = nb_blocks.strict_mul(block_len); + let nb_stripes = remaining.strict_sub(block_offset) / STRIPE_LEN; + acc = accumulate_loop(acc, input, block_offset, secret, 0, nb_stripes); - accumulate_512( - acc, - &input[input.len() - STRIPE_LEN..], - &secret[secret.len() - STRIPE_LEN - SECRET_LASTACC_START..secret.len() - SECRET_LASTACC_START], - ) + let input_start = input.len().strict_sub(STRIPE_LEN); + let secret_end = secret.len().strict_sub(SECRET_LASTACC_START); + let secret_start = secret_end.strict_sub(STRIPE_LEN); + + accumulate_512(acc, &input[input_start..], &secret[secret_start..secret_end]) } #[inline(never)] +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] fn xxh3_64_long_impl(input: &[u8], secret: &[u8]) -> u64 { let acc = hash_long_internal_loop(input, secret); merge_accs( @@ -532,6 +588,13 @@ fn xxh3_64_long_impl(input: &[u8], secret: &[u8]) -> u64 { } /// Long-path default-seed entry point (>240B) — no seed branch. +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] pub(crate) fn xxh3_64_long_default(input: &[u8]) -> u64 { xxh3_64_long_impl(input, &DEFAULT_SECRET) } @@ -543,23 +606,19 @@ fn custom_default_secret(seed: u64) -> [u8; DEFAULT_SECRET_SIZE] { } let mut result = [0u8; DEFAULT_SECRET_SIZE]; - const NB_ROUNDS: usize = DEFAULT_SECRET_SIZE / 16; - - let mut idx = 0usize; - while idx < NB_ROUNDS { - // SAFETY: idx < NB_ROUNDS = DEFAULT_SECRET_SIZE/16, so idx*16+16 <= DEFAULT_SECRET_SIZE. - let lo = unsafe { read_u64_le(&DEFAULT_SECRET, idx * 16).wrapping_add(seed).to_le_bytes() }; - // SAFETY: idx < NB_ROUNDS = DEFAULT_SECRET_SIZE/16, so idx*16+8+8 <= DEFAULT_SECRET_SIZE. - let hi = unsafe { - read_u64_le(&DEFAULT_SECRET, idx * 16 + 8) - .wrapping_sub(seed) - .to_le_bytes() - }; - - result[idx * 16..idx * 16 + 8].copy_from_slice(&lo); - result[idx * 16 + 8..idx * 16 + 16].copy_from_slice(&hi); - - idx += 1; + let (source_chunks, source_tail) = DEFAULT_SECRET.as_chunks::<16>(); + let (destination_chunks, destination_tail) = result.as_chunks_mut::<16>(); + debug_assert!(source_tail.is_empty()); + debug_assert!(destination_tail.is_empty()); + + for (source, destination) in source_chunks.iter().zip(destination_chunks) { + let (source_words, source_tail) = source.as_chunks::<8>(); + let (destination_words, destination_tail) = destination.as_chunks_mut::<8>(); + debug_assert!(source_tail.is_empty()); + debug_assert!(destination_tail.is_empty()); + + destination_words[0] = u64::from_le_bytes(source_words[0]).wrapping_add(seed).to_le_bytes(); + destination_words[1] = u64::from_le_bytes(source_words[1]).wrapping_sub(seed).to_le_bytes(); } result @@ -568,6 +627,13 @@ fn custom_default_secret(seed: u64) -> [u8; DEFAULT_SECRET_SIZE] { /// Long-path entry point (>240B) — no ≤240B branches. /// /// Called from compile-time dispatch when the caller already knows `input.len() > MID_SIZE_MAX`. +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] pub(crate) fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { if seed == 0 { xxh3_64_long_default(input) @@ -584,7 +650,8 @@ fn xxh3_128_1to3(input: &[u8], seed: u64, secret: &[u8]) -> u128 { let c1 = *input.get_unchecked(0); let c2 = *input.get_unchecked(input.len() >> 1); let c3 = *input.get_unchecked(input.len().strict_sub(1)); - let input_lo = (c1 as u32) << 16 | (c2 as u32) << 24 | c3 as u32 | (input.len() as u32) << 8; + let input_len = u32::try_from(input.len()).expect("XXH3 1-to-3-byte path length must fit u32"); + let input_lo = (u32::from(c1) << 16) | (u32::from(c2) << 24) | u32::from(c3) | (input_len << 8); let input_hi = input_lo.swap_bytes().rotate_left(13); let flip_lo = (read_u32_le(secret, 0) as u64 ^ read_u32_le(secret, 4) as u64).wrapping_add(seed); @@ -598,12 +665,12 @@ fn xxh3_128_1to3(input: &[u8], seed: u64, secret: &[u8]) -> u128 { #[inline(always)] fn xxh3_128_4to8(input: &[u8], mut seed: u64, secret: &[u8]) -> u128 { - seed ^= ((seed as u32).swap_bytes() as u64) << 32; + seed ^= u64::from(low_u32(seed).swap_bytes()) << 32; // SAFETY: input.len() is 4..=8, secret.len() >= SECRET_SIZE_MIN (136). unsafe { let lo = read_u32_le(input, 0); - let hi = read_u32_le(input, input.len() - 4); + let hi = read_u32_le(input, input.len().strict_sub(4)); let input_64 = (lo as u64).wrapping_add((hi as u64) << 32); let flip = (read_u64_le(secret, 16) ^ read_u64_le(secret, 24)).wrapping_add(seed); @@ -629,13 +696,13 @@ fn xxh3_128_9to16(input: &[u8], seed: u64, secret: &[u8]) -> u128 { let flip_lo = (read_u64_le(secret, 32) ^ read_u64_le(secret, 40)).wrapping_sub(seed); let flip_hi = (read_u64_le(secret, 48) ^ read_u64_le(secret, 56)).wrapping_add(seed); let input_lo = read_u64_le(input, 0); - let mut input_hi = read_u64_le(input, input.len() - 8); + let mut input_hi = read_u64_le(input, input.len().strict_sub(8)); let (mut mul_low, mut mul_high) = mul64_to128(input_lo ^ input_hi ^ flip_lo, PRIME64_1); - mul_low = mul_low.wrapping_add(((input.len() as u64) - 1) << 54); + mul_low = mul_low.wrapping_add((input.len() as u64).strict_sub(1) << 54); input_hi ^= flip_hi; - mul_high = mul_high.wrapping_add(input_hi.wrapping_add(mult32_to64(input_hi as u32, PRIME32_2 - 1))); + mul_high = mul_high.wrapping_add(input_hi.wrapping_add(mult32_to64(low_u32(input_hi), PRIME32_2.strict_sub(1)))); mul_low ^= mul_high.swap_bytes(); @@ -679,7 +746,7 @@ fn xxh3_128_7to128(input: &[u8], seed: u64, secret: &[u8]) -> u128 { &mut lo, &mut hi, chunk16(input, 48), - chunk16(input, input.len() - 64), + chunk16(input, input.len().strict_sub(64)), chunk32(secret, 96), seed, ); @@ -689,7 +756,7 @@ fn xxh3_128_7to128(input: &[u8], seed: u64, secret: &[u8]) -> u128 { &mut lo, &mut hi, chunk16(input, 32), - chunk16(input, input.len() - 48), + chunk16(input, input.len().strict_sub(48)), chunk32(secret, 64), seed, ); @@ -699,7 +766,7 @@ fn xxh3_128_7to128(input: &[u8], seed: u64, secret: &[u8]) -> u128 { &mut lo, &mut hi, chunk16(input, 16), - chunk16(input, input.len() - 32), + chunk16(input, input.len().strict_sub(32)), chunk32(secret, 32), seed, ); @@ -709,7 +776,7 @@ fn xxh3_128_7to128(input: &[u8], seed: u64, secret: &[u8]) -> u128 { &mut lo, &mut hi, chunk16(input, 0), - chunk16(input, input.len() - 16), + chunk16(input, input.len().strict_sub(16)), chunk32(secret, 0), seed, ); @@ -808,38 +875,40 @@ fn xxh3_128_129to240(input: &[u8], seed: u64, secret: &[u8]) -> u128 { let mut idx = 0usize; while idx < 4 { + let offset = idx.strict_mul(32); mix32_b( &mut lo, &mut hi, - chunk16(input, 32 * idx), - chunk16(input, (32 * idx) + 16), - chunk32(secret, 32 * idx), + chunk16(input, offset), + chunk16(input, offset.strict_add(16)), + chunk32(secret, offset), seed, ); - idx += 1; + idx = idx.strict_add(1); } lo = xxh3_avalanche(lo); hi = xxh3_avalanche(hi); while idx < nb_rounds { - let sec_off = START_OFFSET.wrapping_add(32 * (idx - 4)); + let input_offset = idx.strict_mul(32); + let secret_offset = START_OFFSET.strict_add(idx.strict_sub(4).strict_mul(32)); mix32_b( &mut lo, &mut hi, - chunk16(input, 32 * idx), - chunk16(input, (32 * idx) + 16), - chunk32(secret, sec_off), + chunk16(input, input_offset), + chunk16(input, input_offset.strict_add(16)), + chunk32(secret, secret_offset), seed, ); - idx += 1; + idx = idx.strict_add(1); } mix32_b( &mut lo, &mut hi, - chunk16(input, input.len() - 16), - chunk16(input, input.len() - 32), + chunk16(input, input.len().strict_sub(16)), + chunk16(input, input.len().strict_sub(32)), chunk32(secret, SECRET_SIZE_MIN - LAST_OFFSET - 16), 0u64.wrapping_sub(seed), ); @@ -855,6 +924,13 @@ fn xxh3_128_129to240(input: &[u8], seed: u64, secret: &[u8]) -> u128 { } #[inline(never)] +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] fn xxh3_128_long_impl(input: &[u8], secret: &[u8]) -> u128 { let acc = hash_long_internal_loop(input, secret); @@ -867,7 +943,10 @@ fn xxh3_128_long_impl(input: &[u8], secret: &[u8]) -> u128 { let hi = merge_accs( &acc, secret, - secret.len() - (ACC_NB * mem::size_of::()) - SECRET_MERGEACCS_START, + secret + .len() + .strict_sub(ACC_NB.strict_mul(mem::size_of::())) + .strict_sub(SECRET_MERGEACCS_START), !(input.len() as u64).wrapping_mul(PRIME64_2), ); @@ -875,11 +954,25 @@ fn xxh3_128_long_impl(input: &[u8], secret: &[u8]) -> u128 { } /// Long-path default-seed entry point (>240B) — no seed branch. +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] pub(crate) fn xxh3_128_long_default(input: &[u8]) -> u128 { xxh3_128_long_impl(input, &DEFAULT_SECRET) } /// Long-path entry point (>240B) — no ≤240B branches. +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] pub(crate) fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { xxh3_128_long_default(input) @@ -1008,6 +1101,18 @@ mod tests { xxhash_rust::xxh3::xxh3_128_with_seed(&data, seed), "xxh3_128 mismatch (seed={seed}, len={len})" ); + if len > super::MID_SIZE_MAX { + assert_eq!( + super::xxh3_64_long(&data, seed), + xxhash_rust::xxh3::xxh3_64_with_seed(&data, seed), + "portable xxh3_64 long-path mismatch (seed={seed}, len={len})" + ); + assert_eq!( + super::xxh3_128_long(&data, seed), + xxhash_rust::xxh3::xxh3_128_with_seed(&data, seed), + "portable xxh3_128 long-path mismatch (seed={seed}, len={len})" + ); + } } } } diff --git a/src/hashes/fast/xxh3/aarch64_neon.rs b/src/hashes/fast/xxh3/aarch64_neon.rs index d9f0b4b9..8e06cbce 100644 --- a/src/hashes/fast/xxh3/aarch64_neon.rs +++ b/src/hashes/fast/xxh3/aarch64_neon.rs @@ -15,8 +15,6 @@ //! //! Uses `unsafe` for NEON intrinsics. Callers must ensure NEON is available //! (always true on aarch64 — it is the baseline ISA). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] use core::arch::aarch64::*; @@ -32,6 +30,9 @@ const LAST_ACC_SECRET_OFFSET: usize = DEFAULT_SECRET_SIZE - STRIPE_LEN - SECRET_ // SIMD accumulate + scramble +/// # Safety +/// +/// The caller must ensure NEON is available, as guaranteed by AArch64. #[inline] #[target_feature(enable = "neon")] unsafe fn load_acc(initial: &[u64; ACC_NB]) -> [uint64x2_t; 4] { @@ -46,6 +47,9 @@ unsafe fn load_acc(initial: &[u64; ACC_NB]) -> [uint64x2_t; 4] { } } +/// # Safety +/// +/// The caller must ensure NEON is available, as guaranteed by AArch64. #[inline] #[target_feature(enable = "neon")] unsafe fn store_acc(acc: &[uint64x2_t; 4]) -> [u64; ACC_NB] { @@ -67,6 +71,10 @@ unsafe fn store_acc(acc: &[uint64x2_t; 4]) -> [u64; ACC_NB] { /// (vs 2 separate `vmovn` + `vshrn`) /// - `vmlal_high_u32` for the second lane (avoids `vget_high_u32` extraction) /// - `vmlal(data_swap, ...)` base to break the acc→vmlal dependency chain +/// +/// # Safety +/// +/// The caller must ensure NEON is available and both pointers are valid for 64 readable bytes. #[inline] #[target_feature(enable = "neon")] unsafe fn accumulate_512(acc: &mut [uint64x2_t; 4], stripe: *const u8, secret: *const u8) { @@ -112,6 +120,10 @@ unsafe fn accumulate_512(acc: &mut [uint64x2_t; 4], stripe: *const u8, secret: * } /// Scramble the accumulator at block boundaries. +/// +/// # Safety +/// +/// The caller must ensure NEON is available and `secret` is valid for 64 readable bytes. #[inline] #[target_feature(enable = "neon")] unsafe fn scramble_acc(acc: &mut [uint64x2_t; 4], secret: *const u8) { @@ -154,11 +166,11 @@ unsafe fn scramble_acc(acc: &mut [uint64x2_t; 4], secret: *const u8) { /// buffer — `prfm` silently ignores invalid addresses. #[inline(always)] #[cfg(miri)] -unsafe fn prefetch_stripe(_input_ptr: *const u8) {} +fn prefetch_stripe(_input_ptr: *const u8) {} #[inline(always)] #[cfg(not(miri))] -unsafe fn prefetch_stripe(input_ptr: *const u8) { +fn prefetch_stripe(input_ptr: *const u8) { // SAFETY: PRFM is a CPU hint; invalid addresses are silently ignored. unsafe { core::arch::asm!( @@ -171,6 +183,10 @@ unsafe fn prefetch_stripe(input_ptr: *const u8) { // Long-path loop (SIMD inner, scalar merge) +/// # Safety +/// +/// The caller must ensure NEON is available, `stripes` is nonzero, and the +/// requested input and secret stripe ranges are in bounds. #[target_feature(enable = "neon")] unsafe fn stream_accumulate_inner( initial: [u64; ACC_NB], @@ -246,8 +262,15 @@ pub(crate) unsafe fn stream_accumulate( } } +/// # Safety +/// +/// The caller must ensure NEON is available and `input` contains an XXH3 long +/// input of more than 240 bytes. #[target_feature(enable = "neon")] -unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { +unsafe fn hash_long_internal_loop( + input: &[u8], + secret: &[u8; DEFAULT_SECRET_SIZE], +) -> [u64; ACC_NB] { // SAFETY: NEON available via target_feature. Input/secret bounds checked by caller. unsafe { let mut acc = load_acc(&INITIAL_ACC); @@ -300,8 +323,12 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[ } #[inline] +/// # Safety +/// +/// The caller must ensure NEON is available and `input` contains an XXH3 long +/// input of more than 240 bytes. #[target_feature(enable = "neon")] -unsafe fn hash_long_internal_loop_for_len(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { +unsafe fn hash_long_internal_loop_for_len(input: &[u8], secret: &[u8; DEFAULT_SECRET_SIZE]) -> [u64; ACC_NB] { // Small long inputs have too few stripes to amortize software prefetch. // Keep PRFM for larger streaming inputs where it helps Apple/Neoverse throughput. // SAFETY: caller upholds the NEON and input/secret bounds contracts. @@ -319,7 +346,7 @@ unsafe fn hash_long_internal_loop_for_len(input: &[u8], secret: &[u8]) -> [u64; /// Long-path entry point (>240B) — no ≤240B branches. /// /// Called from compile-time dispatch when the caller already knows `input.len() > MID_SIZE_MAX`. -pub fn xxh3_64_long_default(input: &[u8]) -> u64 { +pub(crate) fn xxh3_64_long_default(input: &[u8]) -> u64 { // SAFETY: NEON always available on aarch64. let acc = unsafe { if input.len() <= 1024 { @@ -336,7 +363,7 @@ pub fn xxh3_64_long_default(input: &[u8]) -> u64 { ) } -pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { +pub(crate) fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { if seed == 0 { xxh3_64_long_default(input) } else { @@ -359,13 +386,13 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_128_long_default(input: &[u8]) -> u128 { +pub(crate) fn xxh3_128_long_default(input: &[u8]) -> u128 { // SAFETY: NEON always available on aarch64. let acc = unsafe { hash_long_internal_loop_for_len(input, &DEFAULT_SECRET) }; xxh3_128_long_finalize(&acc, &DEFAULT_SECRET, input.len()) } -pub fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { +pub(crate) fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { xxh3_128_long_default(input) } else { diff --git a/src/hashes/fast/xxh3/dispatch.rs b/src/hashes/fast/xxh3/dispatch.rs index 0c1b4be4..e18117d7 100644 --- a/src/hashes/fast/xxh3/dispatch.rs +++ b/src/hashes/fast/xxh3/dispatch.rs @@ -1,25 +1,46 @@ -#![cfg_attr(test, allow(dead_code))] - +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] +use super::kernels::{hash64_long_fn, hash128_long_fn}; use super::{ dispatch_tables::DispatchTable, - kernels::{ - StreamAccumulateFn, Xxh3KernelId, hash64_long_fn, hash128_long_fn, required_caps, - stream_accumulate_fn as kernel_stream_accumulate_fn, - }, + kernels::{StreamAccumulateFn, Xxh3KernelId, required_caps, stream_accumulate_fn as kernel_stream_accumulate_fn}, }; use crate::{backend::cache::OnceCache, platform::Caps}; +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] type Hash64Fn = fn(&[u8], u64) -> u64; +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] type Hash128Fn = fn(&[u8], u64) -> u128; #[derive(Clone, Copy)] struct ActiveDispatch { /// Long-path-only entry for 64-bit hash (>240B, no redundant length checks). + #[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" + )))] long64: Hash64Fn, /// Long-path-only entry for 128-bit hash (>240B, no redundant length checks). + #[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" + )))] long128: Hash128Fn, stream_accumulate: StreamAccumulateFn, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] long_id: Xxh3KernelId, } @@ -57,10 +78,20 @@ fn active() -> ActiveDispatch { let long_id = resolve(table.long, caps); ActiveDispatch { + #[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" + )))] long64: hash64_long_fn(long_id), + #[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" + )))] long128: hash128_long_fn(long_id), stream_accumulate: kernel_stream_accumulate_fn(long_id), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] long_id, } }) @@ -71,7 +102,7 @@ pub(crate) fn stream_accumulate_fn() -> StreamAccumulateFn { active().stream_accumulate } -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] #[inline] #[must_use] fn kernel_id64_for_len(long_id: Xxh3KernelId, caps: Caps, len: usize) -> Xxh3KernelId { @@ -90,18 +121,18 @@ fn kernel_id64_for_len(long_id: Xxh3KernelId, caps: Caps, len: usize) -> Xxh3Ker long_id } -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] #[inline] #[must_use] -pub fn kernel_name64_for_len(len: usize) -> &'static str { +pub(crate) fn kernel_name64_for_len(len: usize) -> &'static str { let d = active(); kernel_id64_for_len(d.long_id, crate::platform::caps(), len).as_str() } -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] #[inline] #[must_use] -pub fn kernel_name128_for_len(len: usize) -> &'static str { +pub(crate) fn kernel_name128_for_len(len: usize) -> &'static str { if len <= super::MID_SIZE_MAX { Xxh3KernelId::Portable.as_str() } else { @@ -119,7 +150,7 @@ pub fn kernel_name128_for_len(len: usize) -> &'static str { /// stays out-of-line so ≤240B paths do not carry the cache/feature lookup. #[inline(always)] #[must_use] -pub fn hash64(data: &[u8]) -> u64 { +pub(crate) fn hash64(data: &[u8]) -> u64 { let len = data.len(); if len == 0 { return super::XXH3_64_EMPTY_DEFAULT; @@ -144,7 +175,7 @@ pub fn hash64(data: &[u8]) -> u64 { #[inline(always)] #[must_use] -pub fn hash128(data: &[u8]) -> u128 { +pub(crate) fn hash128(data: &[u8]) -> u128 { let len = data.len(); if len == 0 { return super::XXH3_128_EMPTY_DEFAULT; @@ -169,7 +200,7 @@ pub fn hash128(data: &[u8]) -> u128 { #[inline(always)] #[must_use] -pub fn hash64_with_seed(seed: u64, data: &[u8]) -> u64 { +pub(crate) fn hash64_with_seed(seed: u64, data: &[u8]) -> u64 { let len = data.len(); if len == 0 && seed == 0 { return super::XXH3_64_EMPTY_DEFAULT; @@ -192,40 +223,49 @@ pub fn hash64_with_seed(seed: u64, data: &[u8]) -> u64 { hash64_long(seed, data) } +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx2"))] #[inline(always)] fn hash64_long_default(data: &[u8]) -> u64 { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx2"))] - { - if use_zen5_xxh3_64_avx2_short_long(data.len()) { - return super::x86_64_avx2::xxh3_64_long_default(data); - } + if use_zen5_xxh3_64_avx2_short_long(data.len()) { + super::x86_64_avx2::xxh3_64_long_default(data) + } else { + super::x86_64_avx512::xxh3_64_long_default(data) } +} - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - { - return super::x86_64_avx512::xxh3_64_long_default(data); - } +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", not(target_feature = "avx2")))] +#[inline(always)] +fn hash64_long_default(data: &[u8]) -> u64 { + super::x86_64_avx512::xxh3_64_long_default(data) +} - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] - { - return super::x86_64_avx2::xxh3_64_long_default(data); - } +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] +#[inline(always)] +fn hash64_long_default(data: &[u8]) -> u64 { + super::x86_64_avx2::xxh3_64_long_default(data) +} - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - return super::aarch64_neon::xxh3_64_long_default(data); - } +#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] +#[inline(always)] +fn hash64_long_default(data: &[u8]) -> u64 { + super::aarch64_neon::xxh3_64_long_default(data) +} - #[cfg(target_arch = "riscv64")] - { - return super::xxh3_64_long_default(data); - } +#[cfg(target_arch = "riscv64")] +#[inline(always)] +fn hash64_long_default(data: &[u8]) -> u64 { + super::xxh3_64_long_default(data) +} - #[allow(unreachable_code)] - { - let d = active(); - (d.long64)(data, 0) - } +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] +#[inline(always)] +fn hash64_long_default(data: &[u8]) -> u64 { + let d = active(); + (d.long64)(data, 0) } /// Long-path dispatch (>240B). @@ -238,45 +278,57 @@ fn hash64_long_default(data: &[u8]) -> u64 { /// Falls back to runtime dispatch when features are unknown at compile time, /// using the dedicated long-path entry point that skips redundant ≤240B length /// checks in the kernel. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx2"))] #[inline(always)] fn hash64_long(seed: u64, data: &[u8]) -> u64 { - // Tier 1: compile-time dispatch — dedicated long entry points skip ≤240B - // branches that are guaranteed dead at this call site. - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx2"))] - { - if use_zen5_xxh3_64_avx2_short_long(data.len()) { - return super::x86_64_avx2::xxh3_64_long(data, seed); - } + if use_zen5_xxh3_64_avx2_short_long(data.len()) { + super::x86_64_avx2::xxh3_64_long(data, seed) + } else { + super::x86_64_avx512::xxh3_64_long(data, seed) } +} - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - { - return super::x86_64_avx512::xxh3_64_long(data, seed); - } +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", not(target_feature = "avx2")))] +#[inline(always)] +fn hash64_long(seed: u64, data: &[u8]) -> u64 { + super::x86_64_avx512::xxh3_64_long(data, seed) +} - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] - { - return super::x86_64_avx2::xxh3_64_long(data, seed); - } +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] +#[inline(always)] +fn hash64_long(seed: u64, data: &[u8]) -> u64 { + super::x86_64_avx2::xxh3_64_long(data, seed) +} - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - return super::aarch64_neon::xxh3_64_long(data, seed); - } +#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] +#[inline(always)] +fn hash64_long(seed: u64, data: &[u8]) -> u64 { + super::aarch64_neon::xxh3_64_long(data, seed) +} - // The retired RVV path lost to portable at 256 B–64 KiB on SpacemiT K1. - // Bypass runtime dispatch on RISC-V. - #[cfg(target_arch = "riscv64")] - { - return super::xxh3_64_long(data, seed); - } +// The retired RVV path lost to portable at 256 B–64 KiB on SpacemiT K1. +// Bypass runtime dispatch on RISC-V. +#[cfg(target_arch = "riscv64")] +#[inline(always)] +fn hash64_long(seed: u64, data: &[u8]) -> u64 { + super::xxh3_64_long(data, seed) +} - // Tier 2: runtime dispatch — dedicated long-path fn pointer, no redundant - // length checks. - #[allow(unreachable_code)] +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] +#[inline(always)] +fn hash64_long(seed: u64, data: &[u8]) -> u64 { hash64_long_runtime(seed, data) } +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] #[inline(never)] fn hash64_long_runtime(seed: u64, data: &[u8]) -> u64 { #[cfg(target_arch = "x86_64")] @@ -293,7 +345,7 @@ fn hash64_long_runtime(seed: u64, data: &[u8]) -> u64 { /// See [`hash64_with_seed`] for the dispatch rationale. #[inline(always)] #[must_use] -pub fn hash128_with_seed(seed: u64, data: &[u8]) -> u128 { +pub(crate) fn hash128_with_seed(seed: u64, data: &[u8]) -> u128 { let len = data.len(); if len == 0 && seed == 0 { return super::XXH3_128_EMPTY_DEFAULT; @@ -316,65 +368,82 @@ pub fn hash128_with_seed(seed: u64, data: &[u8]) -> u128 { hash128_long(seed, data) } +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] #[inline(always)] fn hash128_long_default(data: &[u8]) -> u128 { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - { - return super::x86_64_avx512::xxh3_128_long_default(data); - } + super::x86_64_avx512::xxh3_128_long_default(data) +} - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] - { - return super::x86_64_avx2::xxh3_128_long_default(data); - } +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] +#[inline(always)] +fn hash128_long_default(data: &[u8]) -> u128 { + super::x86_64_avx2::xxh3_128_long_default(data) +} - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - return super::aarch64_neon::xxh3_128_long_default(data); - } +#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] +#[inline(always)] +fn hash128_long_default(data: &[u8]) -> u128 { + super::aarch64_neon::xxh3_128_long_default(data) +} - #[cfg(target_arch = "riscv64")] - { - return super::xxh3_128_long_default(data); - } +#[cfg(target_arch = "riscv64")] +#[inline(always)] +fn hash128_long_default(data: &[u8]) -> u128 { + super::xxh3_128_long_default(data) +} - #[allow(unreachable_code)] - { - let d = active(); - (d.long128)(data, 0) - } +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] +#[inline(always)] +fn hash128_long_default(data: &[u8]) -> u128 { + let d = active(); + (d.long128)(data, 0) } /// See [`hash64_long`] for the compile-time dispatch rationale. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] #[inline(always)] fn hash128_long(seed: u64, data: &[u8]) -> u128 { - // Tier 1: compile-time dispatch (dedicated long entry points). - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - { - return super::x86_64_avx512::xxh3_128_long(data, seed); - } + super::x86_64_avx512::xxh3_128_long(data, seed) +} - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] - { - return super::x86_64_avx2::xxh3_128_long(data, seed); - } +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] +#[inline(always)] +fn hash128_long(seed: u64, data: &[u8]) -> u128 { + super::x86_64_avx2::xxh3_128_long(data, seed) +} - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - return super::aarch64_neon::xxh3_128_long(data, seed); - } +#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] +#[inline(always)] +fn hash128_long(seed: u64, data: &[u8]) -> u128 { + super::aarch64_neon::xxh3_128_long(data, seed) +} - // RISC-V: see hash64_long comment. - #[cfg(target_arch = "riscv64")] - { - return super::xxh3_128_long(data, seed); - } +// RISC-V: see hash64_long comment. +#[cfg(target_arch = "riscv64")] +#[inline(always)] +fn hash128_long(seed: u64, data: &[u8]) -> u128 { + super::xxh3_128_long(data, seed) +} - // Tier 2: runtime dispatch — dedicated long-path fn pointer. - #[allow(unreachable_code)] +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] +#[inline(always)] +fn hash128_long(seed: u64, data: &[u8]) -> u128 { hash128_long_runtime(seed, data) } +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] #[inline(never)] fn hash128_long_runtime(seed: u64, data: &[u8]) -> u128 { let d = active(); diff --git a/src/hashes/fast/xxh3/dispatch_tables.rs b/src/hashes/fast/xxh3/dispatch_tables.rs index c600d01b..9643a885 100644 --- a/src/hashes/fast/xxh3/dispatch_tables.rs +++ b/src/hashes/fast/xxh3/dispatch_tables.rs @@ -2,15 +2,15 @@ //! //! This module is the checked-in runtime table used by capability-driven dispatch. -pub use super::kernels::Xxh3KernelId as KernelId; +pub(crate) use super::kernels::Xxh3KernelId as KernelId; use crate::platform::Caps; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub long: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { long: KernelId::Portable, }; @@ -18,27 +18,27 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { /// x86-64 with AVX-512F: single-iteration per stripe. #[cfg(target_arch = "x86_64")] -pub static AVX512_TABLE: DispatchTable = DispatchTable { long: KernelId::Avx512 }; +pub(crate) static AVX512_TABLE: DispatchTable = DispatchTable { long: KernelId::Avx512 }; /// x86-64 with AVX2 (no AVX-512): two iterations per stripe. #[cfg(target_arch = "x86_64")] -pub static AVX2_TABLE: DispatchTable = DispatchTable { long: KernelId::Avx2 }; +pub(crate) static AVX2_TABLE: DispatchTable = DispatchTable { long: KernelId::Avx2 }; /// aarch64 with NEON: four iterations per stripe. #[cfg(target_arch = "aarch64")] -pub static NEON_TABLE: DispatchTable = DispatchTable { long: KernelId::Neon }; +pub(crate) static NEON_TABLE: DispatchTable = DispatchTable { long: KernelId::Neon }; /// POWER8+ with VSX: four iterations per stripe (128-bit vectors). #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] -pub static VSX_TABLE: DispatchTable = DispatchTable { long: KernelId::Vsx }; +pub(crate) static VSX_TABLE: DispatchTable = DispatchTable { long: KernelId::Vsx }; /// s390x z13+ with z/Vector: four iterations per stripe (128-bit vectors). #[cfg(target_arch = "s390x")] -pub static ZVECTOR_TABLE: DispatchTable = DispatchTable { long: KernelId::Vector }; +pub(crate) static ZVECTOR_TABLE: DispatchTable = DispatchTable { long: KernelId::Vector }; #[inline] #[must_use] -pub fn select_runtime_table(caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { let _ = caps; #[cfg(target_arch = "x86_64")] { diff --git a/src/hashes/fast/xxh3/kernels.rs b/src/hashes/fast/xxh3/kernels.rs index 99f30202..f47c06f1 100644 --- a/src/hashes/fast/xxh3/kernels.rs +++ b/src/hashes/fast/xxh3/kernels.rs @@ -1,12 +1,12 @@ use super::{ACC_NB, DEFAULT_SECRET_SIZE}; use crate::platform::Caps; -pub type StreamAccumulateFn = +pub(crate) type StreamAccumulateFn = unsafe fn([u64; ACC_NB], &[u8], usize, &[u8; DEFAULT_SECRET_SIZE], usize, usize, bool) -> [u64; ACC_NB]; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] -pub enum Xxh3KernelId { +pub(crate) enum Xxh3KernelId { Portable = 0, #[cfg(target_arch = "x86_64")] Avx2 = 1, @@ -21,10 +21,10 @@ pub enum Xxh3KernelId { } impl Xxh3KernelId { - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "x86_64")] @@ -42,8 +42,16 @@ impl Xxh3KernelId { } /// Long-path-only entry for 64-bit hash (>240B, no ≤240B length checks). +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" + )) +))] #[must_use] -pub fn hash64_long_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u64 { +pub(crate) fn hash64_long_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u64 { match id { Xxh3KernelId::Portable => super::xxh3_64_long, #[cfg(target_arch = "x86_64")] @@ -60,8 +68,16 @@ pub fn hash64_long_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u64 { } /// Long-path-only entry for 128-bit hash (>240B, no ≤240B length checks). +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" + )) +))] #[must_use] -pub fn hash128_long_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u128 { +pub(crate) fn hash128_long_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u128 { match id { Xxh3KernelId::Portable => super::xxh3_128_long, #[cfg(target_arch = "x86_64")] @@ -78,7 +94,7 @@ pub fn hash128_long_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u128 { } #[must_use] -pub fn stream_accumulate_fn(id: Xxh3KernelId) -> StreamAccumulateFn { +pub(crate) fn stream_accumulate_fn(id: Xxh3KernelId) -> StreamAccumulateFn { match id { Xxh3KernelId::Portable => super::stream_accumulate_portable, #[cfg(target_arch = "x86_64")] @@ -96,7 +112,7 @@ pub fn stream_accumulate_fn(id: Xxh3KernelId) -> StreamAccumulateFn { #[inline] #[must_use] -pub const fn required_caps(id: Xxh3KernelId) -> Caps { +pub(crate) const fn required_caps(id: Xxh3KernelId) -> Caps { match id { Xxh3KernelId::Portable => Caps::NONE, #[cfg(target_arch = "x86_64")] diff --git a/src/hashes/fast/xxh3/power.rs b/src/hashes/fast/xxh3/power.rs index bae33282..25b0f50b 100644 --- a/src/hashes/fast/xxh3/power.rs +++ b/src/hashes/fast/xxh3/power.rs @@ -8,8 +8,6 @@ //! Uses `unsafe` for POWER8+ VSX inline asm. Callers must ensure POWER8+ //! vector support before executing the accelerated path (the dispatcher //! does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] use core::simd::i64x2; @@ -21,6 +19,10 @@ use super::{ // VSX primitive operations (inline asm, POWER8+) /// Add u64 lanes: `vaddudm`. +/// +/// # Safety +/// +/// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn vadd_u64(a: i64x2, b: i64x2) -> i64x2 { @@ -39,6 +41,10 @@ unsafe fn vadd_u64(a: i64x2, b: i64x2) -> i64x2 { } /// Logical shift right u64 lanes: `vsrd`. +/// +/// # Safety +/// +/// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn vshr_u64(a: i64x2, shift: i64x2) -> i64x2 { @@ -57,6 +63,10 @@ unsafe fn vshr_u64(a: i64x2, shift: i64x2) -> i64x2 { } /// Shift left u64 lanes: `vsld`. +/// +/// # Safety +/// +/// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn vshl_u64(a: i64x2, shift: i64x2) -> i64x2 { @@ -82,6 +92,10 @@ unsafe fn vshl_u64(a: i64x2, shift: i64x2) -> i64x2 { /// /// (Using `vmuleuw` here would multiply the ISA "even" = **high** 32 bits, /// which is wrong for the XXH3 MAC pattern on LE.) +/// +/// # Safety +/// +/// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn vmul_low32(a: i64x2, b: i64x2) -> i64x2 { @@ -102,17 +116,25 @@ unsafe fn vmul_low32(a: i64x2, b: i64x2) -> i64x2 { // Load, store, swap /// Load 128 bits from memory (unaligned). +/// +/// # Safety +/// +/// `ptr` must be valid for 16 readable bytes. #[inline(always)] unsafe fn vload(ptr: *const u8) -> i64x2 { // SAFETY: caller ensures ptr is valid for 16 bytes. - unsafe { core::ptr::read_unaligned(ptr as *const i64x2) } + unsafe { core::ptr::read_unaligned(ptr.cast::()) } } /// Store 128 bits to memory (unaligned). +/// +/// # Safety +/// +/// `ptr` must be valid for 16 writable bytes. #[inline(always)] unsafe fn vstore(ptr: *mut u8, val: i64x2) { // SAFETY: caller ensures ptr is valid for 16 bytes. - unsafe { core::ptr::write_unaligned(ptr as *mut i64x2, val) } + unsafe { core::ptr::write_unaligned(ptr.cast::(), val) } } /// Swap u64 lanes (idx ^ 1 effect). @@ -123,23 +145,29 @@ fn vswap(a: i64x2) -> i64x2 { // SIMD accumulate + scramble +/// # Safety +/// +/// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn load_acc(initial: &[u64; ACC_NB]) -> [i64x2; 4] { // SAFETY: POWER8+ VSX via target_feature. Pointer valid for 8 × u64. unsafe { - let p = initial.as_ptr() as *const u8; + let p = initial.as_ptr().cast::(); [vload(p), vload(p.add(16)), vload(p.add(32)), vload(p.add(48))] } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn store_acc(acc: &[i64x2; 4]) -> [u64; ACC_NB] { // SAFETY: POWER8+ VSX via target_feature. unsafe { let mut out = [0u64; ACC_NB]; - let p = out.as_mut_ptr() as *mut u8; + let p = out.as_mut_ptr().cast::(); vstore(p, acc[0]); vstore(p.add(16), acc[1]); vstore(p.add(32), acc[2]); @@ -156,6 +184,11 @@ unsafe fn store_acc(acc: &[i64x2; 4]) -> [u64; ACC_NB] { /// 3. `vmuleuw`: low32(data_key) × high32(data_key) → u64 /// 4. Swap u64 lanes (idx ^ 1) and add data /// 5. Accumulate product + swapped data into acc +/// +/// # Safety +/// +/// The caller must ensure the POWER vector features are available and both +/// pointers are valid for 64 readable bytes. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn accumulate_512(acc: &mut [i64x2; 4], stripe: *const u8, secret: *const u8) { @@ -188,6 +221,11 @@ unsafe fn accumulate_512(acc: &mut [i64x2; 4], stripe: *const u8, secret: *const /// /// Per element: `acc = (xorshift64(acc, 47) ^ secret) * PRIME32_1` /// The 64-bit multiply by a 32-bit prime is split into lo + hi halves. +/// +/// # Safety +/// +/// The caller must ensure the POWER vector features are available and `secret` +/// is valid for 64 readable bytes. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn scramble_acc(acc: &mut [i64x2; 4], secret: *const u8) { @@ -222,6 +260,10 @@ unsafe fn scramble_acc(acc: &mut [i64x2; 4], secret: *const u8) { // Long-path loop (SIMD inner, scalar merge) +/// # Safety +/// +/// The caller must ensure the POWER vector features are available, `stripes` is +/// nonzero, and the requested input and secret stripe ranges are in bounds. #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn stream_accumulate_inner( initial: [u64; ACC_NB], @@ -298,15 +340,19 @@ pub(crate) unsafe fn stream_accumulate( } } +/// # Safety +/// +/// The caller must ensure the POWER vector features are available and `input` +/// contains an XXH3 long input of more than 240 bytes. #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] -unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { +unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8; DEFAULT_SECRET_SIZE]) -> [u64; ACC_NB] { // SAFETY: POWER8+ VSX via target_feature. Input/secret bounds checked by caller. unsafe { let mut acc = load_acc(&INITIAL_ACC); let nb_stripes = (secret.len().strict_sub(STRIPE_LEN)) / SECRET_CONSUME_RATE; let block_len = STRIPE_LEN.strict_mul(nb_stripes); - let nb_blocks = (input.len().strict_sub(1)) / block_len; + let nb_blocks = input.len().strict_sub(1).strict_div(block_len); let mut block = 0usize; while block < nb_blocks { @@ -349,7 +395,7 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] // Top-level kernel functions (safe wrappers) /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { +pub(crate) fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { if seed == 0 { // SAFETY: Dispatcher verifies POWER8+ VSX before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; @@ -373,7 +419,7 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { +pub(crate) fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { // SAFETY: Dispatcher verifies POWER8+ VSX before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; diff --git a/src/hashes/fast/xxh3/s390x.rs b/src/hashes/fast/xxh3/s390x.rs index cb387664..b809ed7f 100644 --- a/src/hashes/fast/xxh3/s390x.rs +++ b/src/hashes/fast/xxh3/s390x.rs @@ -11,8 +11,6 @@ //! Uses `unsafe` for z/Vector inline asm. Callers must ensure z13+ //! vector facility before executing the accelerated path (the dispatcher //! does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] use core::simd::i64x2; @@ -30,6 +28,10 @@ const BSWAP_MASK: [u8; 16] = [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, // z/Vector primitive operations (inline asm, z13+) /// Add u64 lanes: `vag`. +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vag(a: i64x2, b: i64x2) -> i64x2 { @@ -48,6 +50,10 @@ unsafe fn vag(a: i64x2, b: i64x2) -> i64x2 { } /// Logical shift right u64 lanes by immediate: `vesrlg`. +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vesrlg(a: i64x2) -> i64x2 { @@ -66,6 +72,10 @@ unsafe fn vesrlg(a: i64x2) -> i64x2 { } /// Shift left u64 lanes by immediate: `veslg`. +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn veslg(a: i64x2) -> i64x2 { @@ -87,6 +97,10 @@ unsafe fn veslg(a: i64x2) -> i64x2 { /// /// On s390x (big-endian), odd u32 elements are the low 32 bits of each /// u64 lane. This gives: `low32(a) × low32(b) → u64` per lane. +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vmlof(a: i64x2, b: i64x2) -> i64x2 { @@ -108,6 +122,10 @@ unsafe fn vmlof(a: i64x2, b: i64x2) -> i64x2 { /// /// Selects bytes from the concatenation of `a:a` according to `mask`. /// Used to byte-reverse each u64 element (BE → LE). +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vperm(a: i64x2, mask: i64x2) -> i64x2 { @@ -128,6 +146,10 @@ unsafe fn vperm(a: i64x2, mask: i64x2) -> i64x2 { /// Swap u64 lanes (idx ^ 1 effect): `vpdi` with M3=4. /// /// Result element 0 = source element 1, result element 1 = source element 0. +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vpdi_swap(a: i64x2) -> i64x2 { @@ -147,20 +169,33 @@ unsafe fn vpdi_swap(a: i64x2) -> i64x2 { // Load / store helpers /// Load 128 bits from memory (unaligned, native byte order). +/// +/// # Safety +/// +/// `ptr` must be valid for 16 readable bytes. #[inline(always)] unsafe fn vload_raw(ptr: *const u8) -> i64x2 { // SAFETY: caller ensures ptr is valid for 16 bytes. - unsafe { core::ptr::read_unaligned(ptr as *const i64x2) } + unsafe { core::ptr::read_unaligned(ptr.cast::()) } } /// Store 128 bits to memory (unaligned, native byte order). +/// +/// # Safety +/// +/// `ptr` must be valid for 16 writable bytes. #[inline(always)] unsafe fn vstore(ptr: *mut u8, val: i64x2) { // SAFETY: caller ensures ptr is valid for 16 bytes. - unsafe { core::ptr::write_unaligned(ptr as *mut i64x2, val) } + unsafe { core::ptr::write_unaligned(ptr.cast::(), val) } } /// Load 128 bits with per-element byte-reversal (BE → LE). +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available and `ptr` is valid +/// for 16 readable bytes. #[inline] #[target_feature(enable = "vector")] unsafe fn vload_le(ptr: *const u8, bswap: i64x2) -> i64x2 { @@ -170,20 +205,23 @@ unsafe fn vload_le(ptr: *const u8, bswap: i64x2) -> i64x2 { /// Load the byte-swap permutation mask into a vector register. #[inline(always)] -unsafe fn load_bswap_mask() -> i64x2 { +fn load_bswap_mask() -> i64x2 { // SAFETY: BSWAP_MASK is a 16-byte constant. unsafe { vload_raw(BSWAP_MASK.as_ptr()) } } // SIMD accumulate + scramble +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn load_acc(initial: &[u64; ACC_NB]) -> [i64x2; 4] { // SAFETY: z13+ vector facility via target_feature. Pointer valid for 8 × u64. // Accumulator values are native u64s — no byte-swap needed. unsafe { - let p = initial.as_ptr() as *const u8; + let p = initial.as_ptr().cast::(); [ vload_raw(p), vload_raw(p.add(16)), @@ -193,6 +231,9 @@ unsafe fn load_acc(initial: &[u64; ACC_NB]) -> [i64x2; 4] { } } +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn store_acc(acc: &[i64x2; 4]) -> [u64; ACC_NB] { @@ -200,7 +241,7 @@ unsafe fn store_acc(acc: &[i64x2; 4]) -> [u64; ACC_NB] { // Accumulator values are native u64s — no byte-swap needed. unsafe { let mut out = [0u64; ACC_NB]; - let p = out.as_mut_ptr() as *mut u8; + let p = out.as_mut_ptr().cast::(); vstore(p, acc[0]); vstore(p.add(16), acc[1]); vstore(p.add(32), acc[2]); @@ -217,6 +258,11 @@ unsafe fn store_acc(acc: &[i64x2; 4]) -> [u64; ACC_NB] { /// 3. `vmlof`: low32(data_key) × high32(data_key) → u64 /// 4. Swap u64 lanes (idx ^ 1) and add data /// 5. Accumulate product + swapped data into acc +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available and both pointers +/// are valid for 64 readable bytes. #[inline] #[target_feature(enable = "vector")] unsafe fn accumulate_512(acc: &mut [i64x2; 4], stripe: *const u8, secret: *const u8) { @@ -250,6 +296,11 @@ unsafe fn accumulate_512(acc: &mut [i64x2; 4], stripe: *const u8, secret: *const /// /// Per element: `acc = (xorshift64(acc, 47) ^ secret) * PRIME32_1` /// The 64-bit multiply by a 32-bit prime is split into lo + hi halves. +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available and `secret` is +/// valid for 64 readable bytes. #[inline] #[target_feature(enable = "vector")] unsafe fn scramble_acc(acc: &mut [i64x2; 4], secret: *const u8) { @@ -283,6 +334,10 @@ unsafe fn scramble_acc(acc: &mut [i64x2; 4], secret: *const u8) { // Long-path loop (SIMD inner, scalar merge) +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available, `stripes` is +/// nonzero, and the requested input and secret stripe ranges are in bounds. #[target_feature(enable = "vector")] unsafe fn stream_accumulate_inner( initial: [u64; ACC_NB], @@ -358,8 +413,12 @@ pub(crate) unsafe fn stream_accumulate( } } +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available and `input` +/// contains an XXH3 long input of more than 240 bytes. #[target_feature(enable = "vector")] -unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { +unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8; DEFAULT_SECRET_SIZE]) -> [u64; ACC_NB] { // SAFETY: z13+ vector facility via target_feature. Input/secret bounds // checked by caller. unsafe { @@ -367,7 +426,7 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] let nb_stripes = (secret.len().strict_sub(STRIPE_LEN)) / SECRET_CONSUME_RATE; let block_len = STRIPE_LEN.strict_mul(nb_stripes); - let nb_blocks = (input.len().strict_sub(1)) / block_len; + let nb_blocks = input.len().strict_sub(1).strict_div(block_len); let mut block = 0usize; while block < nb_blocks { @@ -410,7 +469,7 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] // Top-level kernel functions (safe wrappers) /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { +pub(crate) fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { if seed == 0 { // SAFETY: Dispatcher verifies z13+ vector facility before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; @@ -434,7 +493,7 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { +pub(crate) fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { // SAFETY: Dispatcher verifies z13+ vector facility before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; diff --git a/src/hashes/fast/xxh3/stream.rs b/src/hashes/fast/xxh3/stream.rs index 60f76da2..dd81a380 100644 --- a/src/hashes/fast/xxh3/stream.rs +++ b/src/hashes/fast/xxh3/stream.rs @@ -203,14 +203,18 @@ impl Xxh3Hasher { *buffered = 0; } - let direct_len = input.len().saturating_sub(1) / INTERNAL_BUFFER_SIZE * INTERNAL_BUFFER_SIZE; + let direct_len = input + .len() + .saturating_sub(1) + .strict_div(INTERNAL_BUFFER_SIZE) + .strict_mul(INTERNAL_BUFFER_SIZE); let processed_direct = direct_len != 0; if processed_direct { let kernel = *stream_accumulate.get_or_insert_with(dispatch::stream_accumulate_fn); (*acc, *accumulated_stripes) = consume_stripes( kernel, *acc, - direct_len / STRIPE_LEN, + direct_len.strict_div(STRIPE_LEN), *accumulated_stripes, &input[..direct_len], secret, @@ -459,7 +463,11 @@ mod tests { use crate::traits::FastHash; fn data(len: usize) -> Vec { - (0..len).map(|i| i.wrapping_mul(131).wrapping_add(17) as u8).collect() + (u8::MIN..=u8::MAX) + .cycle() + .take(len) + .map(|i| i.wrapping_mul(131).wrapping_add(17)) + .collect() } #[test] @@ -675,7 +683,7 @@ mod tests { } #[test] - #[should_panic] + #[should_panic(expected = "assertion failed: count.strict_mul(STRIPE_LEN) <= input.len()")] fn consume_stripes_rejects_short_input_before_kernel_call() { use super::super::kernels::{Xxh3KernelId, stream_accumulate_fn}; diff --git a/src/hashes/fast/xxh3/x86_64_avx2.rs b/src/hashes/fast/xxh3/x86_64_avx2.rs index 42e2eedf..b3f26886 100644 --- a/src/hashes/fast/xxh3/x86_64_avx2.rs +++ b/src/hashes/fast/xxh3/x86_64_avx2.rs @@ -7,8 +7,6 @@ //! //! Uses `unsafe` for AVX2 intrinsics. Callers must ensure AVX2 is available //! before executing the accelerated path (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] use core::arch::x86_64::*; @@ -27,24 +25,32 @@ const SWAP32: i32 = 0x4E; #[inline] #[target_feature(enable = "avx2")] +/// # Safety +/// +/// The current CPU must support AVX2. unsafe fn load_acc(initial: &[u64; ACC_NB]) -> [__m256i; 2] { - // SAFETY: AVX2 available via target_feature. Pointer valid for 8 × u64. + // SAFETY: The function attribute enables the unaligned AVX2 loads, and + // `initial` contains the eight readable lanes loaded here. unsafe { [ - _mm256_loadu_si256(initial.as_ptr() as *const __m256i), - _mm256_loadu_si256(initial.as_ptr().add(4) as *const __m256i), + _mm256_loadu_si256(initial.as_ptr().cast()), + _mm256_loadu_si256(initial.as_ptr().add(4).cast()), ] } } #[inline] #[target_feature(enable = "avx2")] +/// # Safety +/// +/// The current CPU must support AVX2. unsafe fn store_acc(acc: &[__m256i; 2]) -> [u64; ACC_NB] { - // SAFETY: AVX2 available via target_feature. + // SAFETY: The function attribute enables the unaligned AVX2 stores, and + // `out` contains the eight writable lanes stored here. unsafe { let mut out = [0u64; ACC_NB]; - _mm256_storeu_si256(out.as_mut_ptr() as *mut __m256i, acc[0]); - _mm256_storeu_si256(out.as_mut_ptr().add(4) as *mut __m256i, acc[1]); + _mm256_storeu_si256(out.as_mut_ptr().cast(), acc[0]); + _mm256_storeu_si256(out.as_mut_ptr().add(4).cast(), acc[1]); out } } @@ -57,6 +63,11 @@ unsafe fn store_acc(acc: &[__m256i; 2]) -> [u64; ACC_NB] { /// 3. `vpmuludq`: multiply low32(data_key) × high32(data_key) → u64 /// 4. Shuffle data to swap u64 pairs (idx ^ 1) /// 5. Accumulate product + swapped data into acc +/// +/// # Safety +/// +/// The current CPU must support AVX2. `stripe` and `secret` must each remain +/// valid to read 64 initialized bytes from one allocation. #[inline] #[target_feature(enable = "avx2")] unsafe fn accumulate_512(acc: &mut [__m256i; 2], stripe: *const u8, secret: *const u8) { @@ -65,8 +76,8 @@ unsafe fn accumulate_512(acc: &mut [__m256i; 2], stripe: *const u8, secret: *con unsafe { let mut i = 0usize; while i < 2 { - let data_vec = _mm256_loadu_si256(stripe.add(i.strict_mul(32)) as *const __m256i); - let key_vec = _mm256_loadu_si256(secret.add(i.strict_mul(32)) as *const __m256i); + let data_vec = _mm256_loadu_si256(stripe.add(i.strict_mul(32)).cast()); + let key_vec = _mm256_loadu_si256(secret.add(i.strict_mul(32)).cast()); let data_key = _mm256_xor_si256(data_vec, key_vec); // u32 × u32 → u64 multiply: low32(data_key) × high32(data_key) @@ -87,13 +98,18 @@ unsafe fn accumulate_512(acc: &mut [__m256i; 2], stripe: *const u8, secret: *con /// /// Per element: `acc = (xorshift64(acc, 47) ^ secret) * PRIME32_1` /// The 64-bit multiply by a 32-bit prime is split into lo + hi halves. +/// +/// # Safety +/// +/// The current CPU must support AVX2. `secret` must remain valid to read 64 +/// initialized bytes from one allocation. #[inline] #[target_feature(enable = "avx2")] unsafe fn scramble_acc(acc: &mut [__m256i; 2], secret: *const u8) { // SAFETY: AVX2 available via target_feature. Caller ensures secret points to // ≥64 valid bytes. unsafe { - let prime32 = _mm256_set1_epi32(PRIME32_1 as i32); + let prime32 = _mm256_set1_epi32(PRIME32_1.cast_signed()); let mut i = 0usize; while i < 2 { @@ -101,7 +117,7 @@ unsafe fn scramble_acc(acc: &mut [__m256i; 2], secret: *const u8) { let shifted = _mm256_srli_epi64::<47>(acc_vec); let data_vec = _mm256_xor_si256(acc_vec, shifted); - let key_vec = _mm256_loadu_si256(secret.add(i.strict_mul(32)) as *const __m256i); + let key_vec = _mm256_loadu_si256(secret.add(i.strict_mul(32)).cast()); let data_key = _mm256_xor_si256(data_vec, key_vec); // 64-bit multiply by PRIME32_1: @@ -120,6 +136,10 @@ unsafe fn scramble_acc(acc: &mut [__m256i; 2], secret: *const u8) { // Long-path loop (SIMD inner, scalar merge) #[target_feature(enable = "avx2")] +/// # Safety +/// +/// The current CPU must support AVX2. `stripes` must be nonzero, and the +/// requested input and secret stripes must be within their slices. unsafe fn stream_accumulate_inner( initial: [u64; ACC_NB], input: &[u8], @@ -139,7 +159,8 @@ unsafe fn stream_accumulate_inner( ); // SAFETY: Accumulating validated XXH3 stripes because: // 1. The dispatcher selects this function only when AVX2 is available. - // 2. The assertions above prove every 64-byte input and secret load is in bounds. + // 2. The caller contract keeps every 64-byte input and secret load in bounds; + // the assertions above check that contract in debug builds. // 3. `initial` and the returned array each contain exactly eight `u64` lanes. unsafe { let mut acc = load_acc(&initial); @@ -195,14 +216,18 @@ pub(crate) unsafe fn stream_accumulate( } #[target_feature(enable = "avx2")] -unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { +/// # Safety +/// +/// The current CPU must support AVX2, and `input` must contain more than 240 +/// bytes. +unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8; DEFAULT_SECRET_SIZE]) -> [u64; ACC_NB] { // SAFETY: AVX2 available via target_feature. Input/secret bounds checked by caller. unsafe { let mut acc = load_acc(&INITIAL_ACC); - let nb_stripes = (secret.len().strict_sub(STRIPE_LEN)) / SECRET_CONSUME_RATE; + let nb_stripes = secret.len().strict_sub(STRIPE_LEN).strict_div(SECRET_CONSUME_RATE); let block_len = STRIPE_LEN.strict_mul(nb_stripes); - let nb_blocks = (input.len().strict_sub(1)) / block_len; + let nb_blocks = input.len().strict_sub(1).strict_div(block_len); let mut block = 0usize; while block < nb_blocks { @@ -218,7 +243,11 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] } // Remaining stripes in final partial block - let nb_stripes_final = (input.len().strict_sub(1).strict_sub(block_len.strict_mul(nb_blocks))) / STRIPE_LEN; + let nb_stripes_final = input + .len() + .strict_sub(1) + .strict_sub(block_len.strict_mul(nb_blocks)) + .strict_div(STRIPE_LEN); let mut stripe = 0usize; while stripe < nb_stripes_final { let input_off = nb_blocks @@ -245,7 +274,7 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] // Top-level kernel functions (safe wrappers) /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_64_long_default(input: &[u8]) -> u64 { +pub(crate) fn xxh3_64_long_default(input: &[u8]) -> u64 { // SAFETY: Dispatcher verifies AVX2 before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; super::merge_accs( @@ -256,7 +285,7 @@ pub fn xxh3_64_long_default(input: &[u8]) -> u64 { ) } -pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { +pub(crate) fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { if seed == 0 { xxh3_64_long_default(input) } else { @@ -273,13 +302,13 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_128_long_default(input: &[u8]) -> u128 { +pub(crate) fn xxh3_128_long_default(input: &[u8]) -> u128 { // SAFETY: Dispatcher verifies AVX2 before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; xxh3_128_long_finalize(&acc, &DEFAULT_SECRET, input.len()) } -pub fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { +pub(crate) fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { xxh3_128_long_default(input) } else { diff --git a/src/hashes/fast/xxh3/x86_64_avx512.rs b/src/hashes/fast/xxh3/x86_64_avx512.rs index 13a95ac5..7f87ca4e 100644 --- a/src/hashes/fast/xxh3/x86_64_avx512.rs +++ b/src/hashes/fast/xxh3/x86_64_avx512.rs @@ -7,8 +7,6 @@ //! //! Uses `unsafe` for AVX-512F intrinsics. Callers must ensure AVX-512F is //! available before executing the accelerated path (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] use core::arch::x86_64::*; @@ -27,18 +25,26 @@ const SWAP32: i32 = 0x4E; #[inline] #[target_feature(enable = "avx512f")] +/// # Safety +/// +/// The current CPU must support AVX-512F. unsafe fn load_acc(initial: &[u64; ACC_NB]) -> __m512i { - // SAFETY: AVX-512F available via target_feature. Pointer valid for 8 × u64. - unsafe { _mm512_loadu_si512(initial.as_ptr() as *const __m512i) } + // SAFETY: The function attribute enables the unaligned AVX-512F load, and + // `initial` contains the eight readable lanes loaded here. + unsafe { _mm512_loadu_si512(initial.as_ptr().cast()) } } #[inline] #[target_feature(enable = "avx512f")] +/// # Safety +/// +/// The current CPU must support AVX-512F. unsafe fn store_acc(acc: __m512i) -> [u64; ACC_NB] { - // SAFETY: AVX-512F available via target_feature. + // SAFETY: The function attribute enables the unaligned AVX-512F store, and + // `out` contains the eight writable lanes stored here. unsafe { let mut out = [0u64; ACC_NB]; - _mm512_storeu_si512(out.as_mut_ptr() as *mut __m512i, acc); + _mm512_storeu_si512(out.as_mut_ptr().cast(), acc); out } } @@ -48,14 +54,19 @@ unsafe fn store_acc(acc: __m512i) -> [u64; ACC_NB] { /// Processes all 8 u64 accumulators simultaneously: /// 1. Load full 64 B stripe + 64 B secret in one shot /// 2. XOR → shift → multiply → shuffle-swap → accumulate +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. `stripe` and `secret` must each +/// remain valid to read 64 initialized bytes from one allocation. #[inline] #[target_feature(enable = "avx512f")] unsafe fn accumulate_512(acc: &mut __m512i, stripe: *const u8, secret: *const u8) { // SAFETY: AVX-512F available via target_feature. Caller ensures stripe and // secret point to ≥64 valid bytes. unsafe { - let data_vec = _mm512_loadu_si512(stripe as *const __m512i); - let key_vec = _mm512_loadu_si512(secret as *const __m512i); + let data_vec = _mm512_loadu_si512(stripe.cast()); + let key_vec = _mm512_loadu_si512(secret.cast()); let data_key = _mm512_xor_si512(data_vec, key_vec); // u32 × u32 → u64 multiply: low32(data_key) × high32(data_key) @@ -70,18 +81,23 @@ unsafe fn accumulate_512(acc: &mut __m512i, stripe: *const u8, secret: *const u8 } /// Scramble the accumulator at block boundaries — single iteration. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. `secret` must remain valid to read +/// 64 initialized bytes from one allocation. #[inline] #[target_feature(enable = "avx512f")] unsafe fn scramble_acc(acc: &mut __m512i, secret: *const u8) { // SAFETY: AVX-512F available via target_feature. Caller ensures secret // points to ≥64 valid bytes. unsafe { - let prime32 = _mm512_set1_epi32(PRIME32_1 as i32); + let prime32 = _mm512_set1_epi32(PRIME32_1.cast_signed()); let shifted = _mm512_srli_epi64::<47>(*acc); let data_vec = _mm512_xor_si512(*acc, shifted); - let key_vec = _mm512_loadu_si512(secret as *const __m512i); + let key_vec = _mm512_loadu_si512(secret.cast()); let data_key = _mm512_xor_si512(data_vec, key_vec); // 64-bit multiply by PRIME32_1 (split into lo + hi halves) @@ -95,6 +111,10 @@ unsafe fn scramble_acc(acc: &mut __m512i, secret: *const u8) { // Long-path loop (SIMD inner, scalar merge) #[target_feature(enable = "avx512f")] +/// # Safety +/// +/// The current CPU must support AVX-512F. `stripes` must be nonzero, and the +/// requested input and secret stripes must be within their slices. unsafe fn stream_accumulate_inner( initial: [u64; ACC_NB], input: &[u8], @@ -114,7 +134,8 @@ unsafe fn stream_accumulate_inner( ); // SAFETY: Accumulating validated XXH3 stripes because: // 1. The dispatcher selects this function only when AVX-512F is available. - // 2. The assertions above prove every 64-byte input and secret load is in bounds. + // 2. The caller contract keeps every 64-byte input and secret load in bounds; + // the assertions above check that contract in debug builds. // 3. `initial` and the returned array each contain exactly eight `u64` lanes. unsafe { let mut acc = load_acc(&initial); @@ -170,15 +191,19 @@ pub(crate) unsafe fn stream_accumulate( } #[target_feature(enable = "avx512f")] -unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { +/// # Safety +/// +/// The current CPU must support AVX-512F, and `input` must contain more than +/// 240 bytes. +unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8; DEFAULT_SECRET_SIZE]) -> [u64; ACC_NB] { // SAFETY: AVX-512F available via target_feature. Input/secret bounds checked // by caller. unsafe { let mut acc = load_acc(&INITIAL_ACC); - let nb_stripes = (secret.len().strict_sub(STRIPE_LEN)) / SECRET_CONSUME_RATE; + let nb_stripes = secret.len().strict_sub(STRIPE_LEN).strict_div(SECRET_CONSUME_RATE); let block_len = STRIPE_LEN.strict_mul(nb_stripes); - let nb_blocks = (input.len().strict_sub(1)) / block_len; + let nb_blocks = input.len().strict_sub(1).strict_div(block_len); let mut block = 0usize; while block < nb_blocks { @@ -194,7 +219,11 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] } // Remaining stripes in final partial block - let nb_stripes_final = (input.len().strict_sub(1).strict_sub(block_len.strict_mul(nb_blocks))) / STRIPE_LEN; + let nb_stripes_final = input + .len() + .strict_sub(1) + .strict_sub(block_len.strict_mul(nb_blocks)) + .strict_div(STRIPE_LEN); let mut stripe = 0usize; while stripe < nb_stripes_final { let input_off = nb_blocks @@ -221,7 +250,7 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] // Top-level kernel functions (safe wrappers) /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_64_long_default(input: &[u8]) -> u64 { +pub(crate) fn xxh3_64_long_default(input: &[u8]) -> u64 { // SAFETY: Dispatcher verifies AVX-512F before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; super::merge_accs( @@ -232,7 +261,7 @@ pub fn xxh3_64_long_default(input: &[u8]) -> u64 { ) } -pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { +pub(crate) fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { if seed == 0 { xxh3_64_long_default(input) } else { @@ -249,13 +278,13 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_128_long_default(input: &[u8]) -> u128 { +pub(crate) fn xxh3_128_long_default(input: &[u8]) -> u128 { // SAFETY: Dispatcher verifies AVX-512F before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; xxh3_128_long_finalize(&acc, &DEFAULT_SECRET, input.len()) } -pub fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { +pub(crate) fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { xxh3_128_long_default(input) } else { diff --git a/src/hashes/mod.rs b/src/hashes/mod.rs index 4248c89e..dcc24822 100644 --- a/src/hashes/mod.rs +++ b/src/hashes/mod.rs @@ -45,16 +45,16 @@ //! ```toml //! [dependencies] //! # Smallest SHA-2-only build -//! rscrypto = { version = "0.7", default-features = false, features = ["sha2"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["sha2"] } //! //! # All cryptographic hashes -//! rscrypto = { version = "0.7", default-features = false, features = ["crypto-hashes"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["crypto-hashes"] } //! //! # Fast non-cryptographic hashes only -//! rscrypto = { version = "0.7", default-features = false, features = ["fast-hashes"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["fast-hashes"] } //! //! # Everything hash-related -//! rscrypto = { version = "0.7", default-features = false, features = ["hashes"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["hashes"] } //! ``` //! //! # API Conventions diff --git a/src/hashes/util.rs b/src/hashes/util.rs index f241ec5f..fbef6e85 100644 --- a/src/hashes/util.rs +++ b/src/hashes/util.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] // Fixed-size array indexing and block parsing - #[repr(C, align(64))] pub(crate) struct Aligned64(pub T); @@ -13,11 +11,11 @@ impl core::ops::Deref for Aligned64 { } #[inline(always)] -pub const fn rotr32(x: u32, n: u32) -> u32 { +pub(super) const fn rotr32(x: u32, n: u32) -> u32 { x.rotate_right(n) } #[inline(always)] -pub const fn rotr64(x: u64, n: u32) -> u64 { +pub(super) const fn rotr64(x: u64, n: u32) -> u64 { x.rotate_right(n) } diff --git a/src/hex.rs b/src/hex.rs index 48f0bef3..97676f72 100644 --- a/src/hex.rs +++ b/src/hex.rs @@ -1,18 +1,3 @@ -#![allow(clippy::indexing_slicing)] -#![cfg_attr( - not(any( - feature = "aes-gcm", - feature = "aes-gcm-siv", - feature = "chacha20poly1305", - feature = "xchacha20poly1305", - feature = "aegis256", - feature = "ascon-aead", - feature = "ed25519", - feature = "x25519" - )), - allow(dead_code, unused_macros) -)] - //! Internal hex encoding, decoding, and formatting utilities. //! //! Provides zero-allocation, `no_std`-compatible hex formatting through @@ -22,6 +7,17 @@ use core::fmt; /// Hex decoding error. +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] #[derive(Clone, Copy, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum InvalidHexError { @@ -36,6 +32,17 @@ pub enum InvalidHexError { }, } +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] impl fmt::Debug for InvalidHexError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -48,6 +55,17 @@ impl fmt::Debug for InvalidHexError { } } +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] impl fmt::Display for InvalidHexError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -57,15 +75,37 @@ impl fmt::Display for InvalidHexError { } } +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] impl core::error::Error for InvalidHexError {} /// Decode a single hex character to its 4-bit value. +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] #[inline] const fn decode_nibble(byte: u8) -> Option { match byte { - b'0'..=b'9' => Some(byte - b'0'), - b'a'..=b'f' => Some(byte - b'a' + 10), - b'A'..=b'F' => Some(byte - b'A' + 10), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), _ => None, } } @@ -73,7 +113,18 @@ const fn decode_nibble(byte: u8) -> Option { /// Decode a hex string into `out`. Accepts mixed case, no `0x` prefix. /// /// Returns `InvalidHexError::InvalidLength` when `hex.len() != out.len() * 2`. -pub fn from_hex(hex: &str, out: &mut [u8]) -> Result<(), InvalidHexError> { +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] +pub(crate) fn from_hex(hex: &str, out: &mut [u8]) -> Result<(), InvalidHexError> { let hex = hex.as_bytes(); if hex.len() != out.len().strict_mul(2) { return Err(InvalidHexError::InvalidLength); @@ -107,7 +158,7 @@ pub fn from_hex(hex: &str, out: &mut [u8]) -> Result<(), InvalidHexError> { } /// Write each byte as two lowercase hex characters. -pub fn fmt_hex_lower(bytes: &[u8], f: &mut fmt::Formatter<'_>) -> fmt::Result { +pub(crate) fn fmt_hex_lower(bytes: &[u8], f: &mut fmt::Formatter<'_>) -> fmt::Result { for &b in bytes { write!(f, "{b:02x}")?; } @@ -115,7 +166,18 @@ pub fn fmt_hex_lower(bytes: &[u8], f: &mut fmt::Formatter<'_>) -> fmt::Result { } /// Write each byte as two uppercase hex characters. -pub fn fmt_hex_upper(bytes: &[u8], f: &mut fmt::Formatter<'_>) -> fmt::Result { +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] +pub(crate) fn fmt_hex_upper(bytes: &[u8], f: &mut fmt::Formatter<'_>) -> fmt::Result { for &b in bytes { write!(f, "{b:02X}")?; } @@ -126,14 +188,53 @@ pub fn fmt_hex_upper(bytes: &[u8], f: &mut fmt::Formatter<'_>) -> fmt::Result { /// /// Returned by the `display_secret()` method on secret key types. Implements /// [`Display`](fmt::Display) so you can `format!("{}", key.display_secret())`. +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] pub struct DisplaySecret<'a>(pub(crate) &'a [u8]); +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] impl fmt::Display for DisplaySecret<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt_hex_lower(self.0, f) } } +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] impl fmt::Debug for DisplaySecret<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "DisplaySecret(\"")?; @@ -147,6 +248,17 @@ impl fmt::Debug for DisplaySecret<'_> { /// Implement `LowerHex`, `UpperHex`, `Display`, `Debug`, and `FromStr` for /// a public byte-array newtype that has `as_bytes()`, `from_bytes()`, and /// `LENGTH`. +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] macro_rules! impl_hex_fmt { ($type:ty) => { impl core::fmt::LowerHex for $type { @@ -182,6 +294,17 @@ macro_rules! impl_hex_fmt { /// Implement masked `Debug`, `FromStr`, and `display_secret()` for a secret /// key newtype. Does **not** implement `Display`, `LowerHex`, or `UpperHex` /// to prevent accidental logging of key material. +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] macro_rules! impl_hex_fmt_secret { ($type:ty) => { impl core::str::FromStr for $type { @@ -211,7 +334,20 @@ macro_rules! impl_hex_fmt_secret { /// Implement `serde::Serialize` and `serde::Deserialize` for a byte-array /// newtype with `as_bytes() -> &[u8; N]`, `from_bytes([u8; N]) -> Self`, /// and `LENGTH`. -#[cfg(feature = "serde")] +#[cfg(all( + feature = "serde", + any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" + ) +))] macro_rules! impl_serde_bytes_inner { ($type:ty, $feature:literal) => { #[cfg_attr(docsrs, doc(cfg(feature = $feature)))] @@ -255,7 +391,20 @@ macro_rules! impl_serde_bytes_inner { }; } -#[cfg(feature = "serde")] +#[cfg(all( + feature = "serde", + any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" + ) +))] macro_rules! impl_serde_bytes { ($type:ty) => { impl_serde_bytes_inner!($type, "serde"); @@ -263,14 +412,40 @@ macro_rules! impl_serde_bytes { } // No-op when serde feature is disabled. -#[cfg(not(feature = "serde"))] +#[cfg(all( + not(feature = "serde"), + any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" + ) +))] macro_rules! impl_serde_bytes { ($type:ty) => {}; } /// Implement `serde` for secret material behind the explicit `serde-secrets` /// feature. This keeps broad DTO serialization from silently exporting keys. -#[cfg(feature = "serde-secrets")] +#[cfg(all( + feature = "serde-secrets", + any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" + ) +))] macro_rules! impl_serde_secret_bytes { ($type:ty) => { #[cfg_attr(docsrs, doc(cfg(feature = "serde-secrets")))] @@ -315,13 +490,36 @@ macro_rules! impl_serde_secret_bytes { }; } -#[cfg(not(feature = "serde-secrets"))] +#[cfg(all( + not(feature = "serde-secrets"), + any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" + ) +))] macro_rules! impl_serde_secret_bytes { ($type:ty) => {}; } /// Generate a fallible constructor that fills `Self::LENGTH` bytes from the /// operating system CSPRNG via `getrandom`. +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "x25519", + feature = "xchacha20poly1305" +))] macro_rules! impl_getrandom { () => { /// Try to generate a random instance from the platform entropy source. @@ -344,7 +542,7 @@ macro_rules! impl_getrandom { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_hex_success() -> bool { +pub(crate) fn diag_zeroize_hex_success() -> bool { let parsed = core::hint::black_box("1111111111111111111111111111111111111111111111111111111111111111") .parse::(); core::hint::black_box(parsed.is_ok()) @@ -354,7 +552,7 @@ pub fn diag_zeroize_hex_success() -> bool { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_hex_error() -> bool { +pub(crate) fn diag_zeroize_hex_error() -> bool { let parsed = core::hint::black_box("11111111111111111111111111111111111111111111111111111111111111zz") .parse::(); core::hint::black_box(parsed.is_err()) @@ -362,7 +560,20 @@ pub fn diag_zeroize_hex_error() -> bool { // Tests -#[cfg(test)] +#[cfg(all( + test, + any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" + ) +))] mod tests { use super::*; @@ -373,7 +584,7 @@ mod tests { assert_eq!(hex, "deadbeef"); let mut out = [0u8; 4]; - from_hex(&hex, &mut out).unwrap(); + from_hex(&hex, &mut out).expect("lowercase hex fixture must decode"); assert_eq!(out, bytes); } @@ -391,7 +602,7 @@ mod tests { #[test] fn from_hex_mixed_case() { let mut out = [0u8; 3]; - from_hex("aAbBcC", &mut out).unwrap(); + from_hex("aAbBcC", &mut out).expect("mixed-case hex fixture must decode"); assert_eq!(out, [0xaa, 0xbb, 0xcc]); } @@ -404,7 +615,7 @@ mod tests { #[test] fn from_hex_invalid_char() { let mut out = [0u8; 2]; - let err = from_hex("abzz", &mut out).unwrap_err(); + let err = from_hex("abzz", &mut out).expect_err("non-hex characters must be rejected"); assert_eq!(err, InvalidHexError::InvalidChar { byte: b'z', index: 2 }); } @@ -416,7 +627,9 @@ mod tests { let mut encoded = alloc::string::String::from("11").repeat(Aes256GcmKey::LENGTH); encoded.replace_range(encoded.len().strict_sub(2).., "zz"); assert_eq!( - encoded.parse::().unwrap_err(), + encoded + .parse::() + .expect_err("invalid secret hex must be rejected"), InvalidHexError::InvalidChar { byte: b'z', index: encoded.len().strict_sub(2), diff --git a/src/lib.rs b/src/lib.rs index c583dc1d..d72bb357 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,7 +14,7 @@ //! //! ```toml //! [dependencies] -//! rscrypto = { version = "0.8.0", default-features = false, features = ["sha2"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["sha2"] } //! ``` //! //! # Guides @@ -32,6 +32,8 @@ //! - XOFs: `Type::xof(data)` or `new` / `update` / `finalize_xof`. //! - MACs: `Type::mac(key, data)` and `Type::verify_tag(key, data, tag)`. //! - AEADs: typed keys and nonces, with combined, detached, and `alloc` Vec helpers. +#![deny(missing_debug_implementations, missing_docs)] +#![deny(clippy::print_stderr, clippy::print_stdout)] #![cfg_attr( feature = "sha2", doc = r#" @@ -125,9 +127,6 @@ assert!( //! evidence. See the security guidance for nonce lifecycle, PHC verification //! limits, and platform fallback notes. -#![cfg_attr(not(test), deny(clippy::unwrap_used))] -#![cfg_attr(not(test), deny(clippy::expect_used))] -#![cfg_attr(not(test), deny(clippy::indexing_slicing))] // Exotic-architecture backends require nightly-only features (inline asm + // portable_simd + unstable target-feature flags). Primary targets (x86_64, // aarch64, wasm) compile on stable Rust 1.91.0. @@ -184,7 +183,6 @@ assert!( feature = "crc16", feature = "crc24", feature = "crc32", - feature = "crc64", feature = "blake2b", feature = "blake2s", feature = "blake3", @@ -205,7 +203,6 @@ assert!( feature = "crc16", feature = "crc24", feature = "crc32", - feature = "crc64", feature = "aes-gcm", feature = "aes-gcm-siv", feature = "aegis256" @@ -223,17 +220,16 @@ assert!( #![cfg_attr( all( target_arch = "riscv64", - any(feature = "blake3", feature = "chacha20poly1305", feature = "xchacha20poly1305") + any( + feature = "blake3", + feature = "chacha20poly1305", + feature = "xchacha20poly1305", + feature = "argon2" + ) ), feature(portable_simd) )] -#![cfg_attr( - all( - target_arch = "riscv32", - any(feature = "sha2", feature = "aes-gcm", feature = "aes-gcm-siv", feature = "aegis256") - ), - feature(riscv_ext_intrinsics) -)] +#![cfg_attr(all(target_arch = "riscv32", feature = "sha2"), feature(riscv_ext_intrinsics))] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] @@ -340,20 +336,12 @@ mod secret; ))] pub mod hashes; -#[cfg_attr( - not(any(feature = "kmac", feature = "ascon-hash", feature = "sha3")), - allow(dead_code) -)] #[inline] #[track_caller] +#[cfg(any(test, feature = "kmac", feature = "sha3", feature = "ascon-hash"))] pub(crate) fn bytes_to_bits(len: usize) -> u64 { - let Ok(bytes) = u64::try_from(len) else { - panic!("byte length exceeds u64"); - }; - let Some(bits) = bytes.checked_mul(8) else { - panic!("byte length bit count exceeds u64"); - }; - bits + let bytes = u64::try_from(len).expect("byte length exceeds u64"); + bytes.checked_mul(8).expect("byte length bit count exceeds u64") } // Checksum re-exports. @@ -417,10 +405,10 @@ pub use auth::{Pbkdf2Error, Pbkdf2Params, Pbkdf2Sha256, Pbkdf2Sha512, Pbkdf2Veri pub use auth::{Poly1305, Poly1305OneTimeKey, Poly1305Tag}; #[cfg(feature = "rsa")] pub use auth::{ - RsaEncryptionError, RsaJwtAlgorithm, RsaJwtVerifier, RsaKeyError, RsaKeyGenerationContract, RsaKeyGenerationError, - RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPrivateOpError, RsaPrivateScratch, - RsaProtocolAlgorithmError, RsaPssProfile, RsaPublicExponent, RsaPublicExponentPolicy, RsaPublicKey, - RsaPublicKeyPolicy, RsaPublicOpError, RsaPublicScratch, RsaSignatureProfile, RsaSignatureSigner, + RsaBlindingPair, RsaEncryptionError, RsaJwtAlgorithm, RsaJwtVerifier, RsaKeyError, RsaKeyGenerationContract, + RsaKeyGenerationError, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPrivateOpError, + RsaPrivateScratch, RsaProtocolAlgorithmError, RsaPssProfile, RsaPublicExponent, RsaPublicExponentPolicy, + RsaPublicKey, RsaPublicKeyPolicy, RsaPublicOpError, RsaPublicScratch, RsaSignatureProfile, RsaSignatureSigner, RsaSignatureVerifier, RsaTlsSignatureSchemes, RsaX509PublicKey, RsaX509PublicKeyAlgorithm, }; #[cfg(feature = "scrypt")] @@ -471,8 +459,6 @@ pub use hashes::fast::{Xxh3_128Hasher, Xxh3BuildHasher, Xxh3Hasher}; feature = "xchacha20poly1305", feature = "aegis256", feature = "ascon-aead", - feature = "ecdsa-p256", - feature = "ecdsa-p384", feature = "ed25519", feature = "ml-kem", feature = "x25519" @@ -898,12 +884,14 @@ let third = HmacSha256Tag::from_bytes([0u8; HmacSha256Tag::LENGTH]); let _ = first.ct_eq(&second) == second.ct_eq(&third); ``` -```compile_fail +`Debug` output is deliberately opaque and must not reveal the decision. + +```rust use rscrypto::HmacSha256Tag; let left = HmacSha256Tag::from_bytes([0u8; HmacSha256Tag::LENGTH]); let right = HmacSha256Tag::from_bytes([0u8; HmacSha256Tag::LENGTH]); -let _ = format!("{:?}", left.ct_eq(&right)); +assert_eq!(format!("{:?}", left.ct_eq(&right)), "CtDecision(..)"); ``` ```rust @@ -1192,8 +1180,10 @@ mod direct_io_write_tests { #[test] fn digest_state_accepts_direct_io_write() { let mut digest = Sha256::new(); - digest.write_all(b"hello ").unwrap(); - digest.write_all(b"world").unwrap(); + digest + .write_all(b"hello ") + .expect("in-memory digest write must succeed"); + digest.write_all(b"world").expect("in-memory digest write must succeed"); assert_eq!(digest.finalize(), Sha256::digest(b"hello world")); } @@ -1201,8 +1191,12 @@ mod direct_io_write_tests { #[test] fn checksum_state_accepts_direct_io_write() { let mut checksum = Crc32C::new(); - checksum.write_all(b"hello ").unwrap(); - checksum.write_all(b"world").unwrap(); + checksum + .write_all(b"hello ") + .expect("in-memory checksum write must succeed"); + checksum + .write_all(b"world") + .expect("in-memory checksum write must succeed"); assert_eq!(checksum.finalize(), Crc32C::checksum(b"hello world")); } @@ -1212,7 +1206,9 @@ mod direct_io_write_tests { let mut digest = Sha256::new(); let bufs = [IoSlice::new(b"hello "), IoSlice::new(b"world")]; - let written = digest.write_vectored(&bufs).unwrap(); + let written = digest + .write_vectored(&bufs) + .expect("in-memory vectored digest write must succeed"); assert_eq!(written, b"hello world".len()); assert_eq!(digest.finalize(), Sha256::digest(b"hello world")); @@ -1221,7 +1217,6 @@ mod direct_io_write_tests { #[cfg(test)] mod send_sync_assertions { - #![allow(unused_imports)] use super::*; fn assert_send_sync() {} @@ -1343,14 +1338,14 @@ mod send_sync_assertions { #[cfg(all(feature = "checksums", feature = "std"))] fn io_adapter_types_are_send_and_sync() { // ChecksumReader/Writer are Send+Sync when their inner types are - assert_send_sync::>, Crc32C>>(); + assert_send_sync::>(); assert_send_sync::, Crc32C>>(); } #[test] #[cfg(all(feature = "hashes", feature = "std"))] fn digest_io_adapter_types_are_send_and_sync() { - assert_send_sync::>, Sha256>>(); + assert_send_sync::>(); assert_send_sync::, Sha256>>(); } @@ -1500,14 +1495,14 @@ mod send_sync_assertions { #[test] #[cfg(all(feature = "checksums", feature = "std"))] fn io_adapter_types_are_debug() { - assert_debug::>, Crc32C>>(); + assert_debug::>(); assert_debug::, Crc32C>>(); } #[test] #[cfg(all(feature = "hashes", feature = "std"))] fn digest_io_adapter_types_are_debug() { - assert_debug::>, Sha256>>(); + assert_debug::>(); assert_debug::, Sha256>>(); } } diff --git a/src/macros.rs b/src/macros.rs index a6851b4f..96d785f7 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -9,6 +9,7 @@ macro_rules! define_unit_error { $vis struct $name; impl $name { + /// Construct this error value. #[inline] #[must_use] pub const fn new() -> Self { @@ -37,7 +38,6 @@ macro_rules! define_unit_error { macro_rules! impl_std_io_write_for_checksum { ($type:ty) => { #[cfg(feature = "std")] - #[allow(clippy::std_instead_of_core)] impl std::io::Write for $type { #[inline] fn write(&mut self, buf: &[u8]) -> std::io::Result { @@ -54,9 +54,7 @@ macro_rules! impl_std_io_write_for_checksum { fn write_vectored(&mut self, bufs: &[std::io::IoSlice<'_>]) -> std::io::Result { let mut written = 0usize; for buf in bufs { - written = written - .checked_add(buf.len()) - .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "vectored write length overflow"))?; + written = written.saturating_add(buf.len()); } for buf in bufs { ::update(self, buf.as_ref()); @@ -78,7 +76,6 @@ macro_rules! impl_std_io_write_for_checksum { macro_rules! impl_std_io_write_for_digest { ($type:ty) => { #[cfg(feature = "std")] - #[allow(clippy::std_instead_of_core)] impl std::io::Write for $type { #[inline] fn write(&mut self, buf: &[u8]) -> std::io::Result { @@ -95,9 +92,7 @@ macro_rules! impl_std_io_write_for_digest { fn write_vectored(&mut self, bufs: &[std::io::IoSlice<'_>]) -> std::io::Result { let mut written = 0usize; for buf in bufs { - written = written - .checked_add(buf.len()) - .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "vectored write length overflow"))?; + written = written.saturating_add(buf.len()); } for buf in bufs { ::update(self, buf.as_ref()); @@ -109,7 +104,7 @@ macro_rules! impl_std_io_write_for_digest { } #[cfg(all( - any(target_arch = "aarch64", target_arch = "riscv64", target_arch = "x86_64"), + any(target_arch = "aarch64", target_arch = "x86_64"), any(feature = "chacha20poly1305", feature = "xchacha20poly1305") ))] macro_rules! define_target_feature_forwarder { @@ -128,6 +123,7 @@ macro_rules! define_target_feature_forwarder { } #[target_feature(enable = $feature)] + #[doc = concat!("Forward through the `", $feature, "` target-feature boundary.\n\n# Safety\n\nThe current CPU must support the named target feature. The safe outer wrapper may call this function only after its dispatch invariant has established that capability.")] unsafe fn __target_feature_forwarder_impl($($arg : $arg_ty),*) $(-> $ret)? { // SAFETY: $inner_safety unsafe { $call } @@ -158,8 +154,7 @@ macro_rules! define_sha_family_dispatch { #[derive(Clone, Copy)] struct Entry { compress_blocks: $compress_fn_ty, - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(feature = "diag")] name: &'static str, } @@ -200,22 +195,22 @@ macro_rules! define_sha_family_dispatch { boundaries: table.boundaries, xs: Entry { compress_blocks: $compress_fn(xs_id), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: xs_id.as_str(), }, s: Entry { compress_blocks: $compress_fn(s_id), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: s_id.as_str(), }, m: Entry { compress_blocks: $compress_fn(m_id), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: m_id.as_str(), }, l: Entry { compress_blocks: $compress_fn(l_id), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: l_id.as_str(), }, } @@ -237,11 +232,10 @@ macro_rules! define_sha_family_dispatch { } } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(feature = "diag")] #[inline] #[must_use] - pub fn kernel_name_for_len(len: usize) -> &'static str { + pub(crate) fn kernel_name_for_len(len: usize) -> &'static str { if $compile_time_hw { return $compile_time_name; } @@ -251,7 +245,7 @@ macro_rules! define_sha_family_dispatch { #[inline] #[must_use] - pub fn digest(data: &[u8]) -> [u8; $output_len] { + pub(crate) fn digest(data: &[u8]) -> [u8; $output_len] { if $compile_time_hw { return digest_oneshot(data, $compile_time_best); } @@ -332,8 +326,7 @@ macro_rules! define_blake2_dispatch { struct Resolved { compress: $compress_fn_ty, compress_blocks: $compress_blocks_fn_ty, - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(feature = "diag")] name: &'static str, } @@ -350,7 +343,7 @@ macro_rules! define_blake2_dispatch { return Resolved { compress: $compress_fn(id), compress_blocks: $compress_blocks_fn(id), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }; } @@ -359,7 +352,7 @@ macro_rules! define_blake2_dispatch { Resolved { compress: $compress_fn($portable_kernel), compress_blocks: $compress_blocks_fn($portable_kernel), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: $portable_kernel.as_str(), } } @@ -392,11 +385,10 @@ macro_rules! define_blake2_dispatch { ACTIVE.get_or_init(resolve).compress_blocks } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(feature = "diag")] #[inline] #[must_use] - pub fn kernel_name_for_len(_len: usize) -> &'static str { + pub(super) fn kernel_name_for_len(_len: usize) -> &'static str { if super::kernels::COMPILE_TIME_HW { return compile_time_name(); } @@ -408,8 +400,7 @@ macro_rules! define_blake2_dispatch { ACTIVE.get_or_init(resolve).name } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(feature = "diag")] const fn compile_time_name() -> &'static str { if cfg!(all( target_arch = "x86_64", diff --git a/src/platform/caps.rs b/src/platform/caps.rs index c96bcb84..04ce30d8 100644 --- a/src/platform/caps.rs +++ b/src/platform/caps.rs @@ -152,7 +152,11 @@ impl Caps { #[inline] #[must_use] pub const fn count(self) -> u32 { - self.0[0].count_ones() + self.0[1].count_ones() + self.0[2].count_ones() + self.0[3].count_ones() + self.0[0] + .count_ones() + .strict_add(self.0[1].count_ones()) + .strict_add(self.0[2].count_ones()) + .strict_add(self.0[3].count_ones()) } /// Create a capability set with a single bit set. @@ -222,18 +226,28 @@ impl core::ops::BitOrAssign for Caps { #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum Arch { + /// 64-bit x86 (`target_arch = "x86_64"`). X86_64, + /// 32-bit x86 (`target_arch = "x86"`). X86, + /// 64-bit Arm (`target_arch = "aarch64"`). Aarch64, + /// 32-bit Arm (`target_arch = "arm"`). Arm, + /// 64-bit RISC-V (`target_arch = "riscv64"`). Riscv64, + /// 32-bit RISC-V (`target_arch = "riscv32"`). Riscv32, - /// IBM POWER (Rust target_arch = "powerpc64"). + /// 64-bit IBM POWER (`target_arch = "powerpc64"`). Power, + /// IBM Z (`target_arch = "s390x"`). S390x, + /// 32-bit WebAssembly (`target_arch = "wasm32"`). Wasm32, + /// 64-bit WebAssembly (`target_arch = "wasm64"`). Wasm64, #[default] + /// Any target architecture not represented by another variant. Other, } @@ -334,74 +348,123 @@ pub mod x86 { use super::Caps; // ─── SSE Family ─── + /// Streaming SIMD Extensions 2. pub const SSE2: Caps = Caps::bit(0); + /// Streaming SIMD Extensions 3. pub const SSE3: Caps = Caps::bit(1); + /// Supplemental Streaming SIMD Extensions 3. pub const SSSE3: Caps = Caps::bit(2); + /// Streaming SIMD Extensions 4.1. pub const SSE41: Caps = Caps::bit(3); + /// Streaming SIMD Extensions 4.2. pub const SSE42: Caps = Caps::bit(4); - pub const SSE4A: Caps = Caps::bit(5); // AMD only + /// AMD Streaming SIMD Extensions 4a. + pub const SSE4A: Caps = Caps::bit(5); // ─── AVX Family ─── + /// Advanced Vector Extensions. pub const AVX: Caps = Caps::bit(6); + /// Advanced Vector Extensions 2. pub const AVX2: Caps = Caps::bit(7); + /// Fused multiply-add instructions. pub const FMA: Caps = Caps::bit(8); + /// Half-precision conversion instructions. pub const F16C: Caps = Caps::bit(9); // ─── Crypto Extensions ─── + /// AES New Instructions. pub const AESNI: Caps = Caps::bit(10); + /// Carry-less multiplication of 64-bit operands. pub const PCLMULQDQ: Caps = Caps::bit(11); + /// SHA-1 and SHA-256 instruction extensions. pub const SHA: Caps = Caps::bit(12); + /// SHA-512 instruction extensions. pub const SHA512: Caps = Caps::bit(13); // ─── AVX-512 Foundation ─── + /// AVX-512 Foundation instructions. pub const AVX512F: Caps = Caps::bit(14); + /// AVX-512 Vector Length extensions for 128- and 256-bit operands. pub const AVX512VL: Caps = Caps::bit(15); + /// AVX-512 byte and word instructions. pub const AVX512BW: Caps = Caps::bit(16); + /// AVX-512 doubleword and quadword instructions. pub const AVX512DQ: Caps = Caps::bit(17); + /// AVX-512 conflict-detection instructions. pub const AVX512CD: Caps = Caps::bit(18); // ─── AVX-512 Crypto/Advanced ─── + /// Vector carry-less multiplication of 64-bit operands. pub const VPCLMULQDQ: Caps = Caps::bit(19); + /// Vector AES instructions. pub const VAES: Caps = Caps::bit(20); + /// Galois Field New Instructions. pub const GFNI: Caps = Caps::bit(21); // ─── AVX-512 Extended ─── + /// AVX-512 integer fused multiply-add instructions. pub const AVX512IFMA: Caps = Caps::bit(22); + /// AVX-512 Vector Byte Manipulation Instructions. pub const AVX512VBMI: Caps = Caps::bit(23); + /// AVX-512 Vector Byte Manipulation Instructions 2. pub const AVX512VBMI2: Caps = Caps::bit(24); + /// AVX-512 Vector Neural Network Instructions. pub const AVX512VNNI: Caps = Caps::bit(25); + /// AVX-512 bit-algorithm instructions. pub const AVX512BITALG: Caps = Caps::bit(26); + /// AVX-512 population-count instructions for doubleword and quadword lanes. pub const AVX512VPOPCNTDQ: Caps = Caps::bit(27); + /// AVX-512 pairwise intersection instructions. pub const AVX512VP2INTERSECT: Caps = Caps::bit(28); + /// AVX-512 half-precision floating-point instructions. pub const AVX512FP16: Caps = Caps::bit(29); + /// AVX-512 BFloat16 instructions. pub const AVX512BF16: Caps = Caps::bit(30); // ─── Bit Manipulation ─── + /// Bit Manipulation Instruction Set 1. pub const BMI1: Caps = Caps::bit(31); + /// Bit Manipulation Instruction Set 2. pub const BMI2: Caps = Caps::bit(32); + /// Population-count instruction. pub const POPCNT: Caps = Caps::bit(33); + /// Leading-zero count instruction. pub const LZCNT: Caps = Caps::bit(34); + /// Multi-precision add-carry instruction extensions. pub const ADX: Caps = Caps::bit(35); // ─── AVX10 (unified AVX-512 replacement) ─── + /// Advanced Vector Extensions 10.1. pub const AVX10_1: Caps = Caps::bit(36); + /// Advanced Vector Extensions 10.2. pub const AVX10_2: Caps = Caps::bit(37); // ─── AMX (Advanced Matrix Extensions) ─── + /// Advanced Matrix Extensions tile configuration and movement instructions. pub const AMX_TILE: Caps = Caps::bit(38); + /// Advanced Matrix Extensions BFloat16 operations. pub const AMX_BF16: Caps = Caps::bit(39); + /// Advanced Matrix Extensions 8-bit integer operations. pub const AMX_INT8: Caps = Caps::bit(40); + /// Advanced Matrix Extensions half-precision floating-point operations. pub const AMX_FP16: Caps = Caps::bit(41); + /// Advanced Matrix Extensions complex-number operations. pub const AMX_COMPLEX: Caps = Caps::bit(42); // ─── Miscellaneous ─── + /// Direct-store instruction for register values. pub const MOVDIRI: Caps = Caps::bit(43); + /// Direct-store instruction for 64-byte memory operands. pub const MOVDIR64B: Caps = Caps::bit(44); + /// Instruction-execution serialization. pub const SERIALIZE: Caps = Caps::bit(45); + /// Hardware random-number generation. pub const RDRAND: Caps = Caps::bit(46); + /// Hardware entropy-seed generation. pub const RDSEED: Caps = Caps::bit(47); // ─── APX (Advanced Performance Extensions) ─── + /// Advanced Performance Extensions foundation. pub const APX: Caps = Caps::bit(48); // ─── Vendor Identification ─── @@ -465,73 +528,120 @@ pub mod aarch64 { use super::Caps; // ─── Basic SIMD ─── - pub const NEON: Caps = Caps::bit(64); // Baseline on AArch64 + /// Advanced SIMD (Neon), which is part of the AArch64 baseline. + pub const NEON: Caps = Caps::bit(64); // ─── Crypto Extensions ─── + /// Arm AES instructions. pub const AES: Caps = Caps::bit(65); - pub const PMULL: Caps = Caps::bit(66); // Often bundled with AES + /// Polynomial multiply long instructions. + pub const PMULL: Caps = Caps::bit(66); + /// Arm SHA-1 and SHA-256 instructions. pub const SHA2: Caps = Caps::bit(67); - pub const SHA3: Caps = Caps::bit(68); // Includes EOR3 + /// Arm SHA-3 instructions, including EOR3. + pub const SHA3: Caps = Caps::bit(68); + /// Arm SHA-512 instructions. pub const SHA512: Caps = Caps::bit(69); + /// Arm SM3 instructions. pub const SM3: Caps = Caps::bit(70); + /// Arm SM4 instructions. pub const SM4: Caps = Caps::bit(71); // ─── CRC Extension ─── + /// Arm CRC32 and CRC32C instructions. pub const CRC: Caps = Caps::bit(72); // ─── Additional SIMD ─── + /// Advanced SIMD integer dot-product instructions. pub const DOTPROD: Caps = Caps::bit(73); + /// Advanced SIMD 8-bit integer matrix-multiply instructions. pub const I8MM: Caps = Caps::bit(74); + /// BFloat16 arithmetic instructions. pub const BF16: Caps = Caps::bit(75); + /// Half-precision floating-point arithmetic instructions. pub const FP16: Caps = Caps::bit(76); + /// Floating-point round-to-integer instructions provided by FEAT_FRINTTS. pub const FRINTTS: Caps = Caps::bit(77); // ─── SVE Family ─── + /// Scalable Vector Extension. pub const SVE: Caps = Caps::bit(78); + /// Scalable Vector Extension version 2. pub const SVE2: Caps = Caps::bit(79); + /// SVE2 AES instructions. pub const SVE2_AES: Caps = Caps::bit(80); + /// SVE2 SHA-3 instructions. pub const SVE2_SHA3: Caps = Caps::bit(81); + /// SVE2 SM4 instructions. pub const SVE2_SM4: Caps = Caps::bit(82); + /// SVE2 bit-permutation instructions. pub const SVE2_BITPERM: Caps = Caps::bit(83); - pub const SVE2_PMULL: Caps = Caps::bit(97); // SVE2 PMULL - pub const SVE2_I8MM: Caps = Caps::bit(98); // SVE2 Int8 matmul - pub const SVE2_F32MM: Caps = Caps::bit(99); // SVE2 FP32 matmul - pub const SVE2_F64MM: Caps = Caps::bit(100); // SVE2 FP64 matmul - pub const SVE2_BF16: Caps = Caps::bit(101); // SVE2 BFloat16 - pub const SVE2_EBF16: Caps = Caps::bit(102); // SVE2 Extended BFloat16 + /// SVE2 polynomial multiply long instructions. + pub const SVE2_PMULL: Caps = Caps::bit(97); + /// SVE2 8-bit integer matrix-multiply instructions. + pub const SVE2_I8MM: Caps = Caps::bit(98); + /// SVE single-precision floating-point matrix-multiply instructions. + pub const SVE2_F32MM: Caps = Caps::bit(99); + /// SVE double-precision floating-point matrix-multiply instructions. + pub const SVE2_F64MM: Caps = Caps::bit(100); + /// SVE BFloat16 instructions. + pub const SVE2_BF16: Caps = Caps::bit(101); + /// SVE extended BFloat16 instructions. + pub const SVE2_EBF16: Caps = Caps::bit(102); // ─── Atomics (ARMv8.1+) ─── - pub const LSE: Caps = Caps::bit(84); // Large System Extensions - pub const LSE2: Caps = Caps::bit(85); // ARMv8.4 atomics + /// Large System Extensions atomic instructions. + pub const LSE: Caps = Caps::bit(84); + /// Large System Extensions version 2 atomic instructions. + pub const LSE2: Caps = Caps::bit(85); // ─── Memory Operations ─── - pub const MOPS: Caps = Caps::bit(86); // FEAT_MOPS memcpy acceleration + /// Memory Copy and Memory Set instructions. + pub const MOPS: Caps = Caps::bit(86); // ─── Scalable Matrix Extension ─── + /// Scalable Matrix Extension. pub const SME: Caps = Caps::bit(87); + /// Scalable Matrix Extension version 2. pub const SME2: Caps = Caps::bit(88); // ─── SME2p1 and extended SME features ─── - pub const SME2P1: Caps = Caps::bit(89); // SME version 2.1 - pub const SME_I16I64: Caps = Caps::bit(90); // SME Int16xInt64 - pub const SME_F64F64: Caps = Caps::bit(91); // SME Float64xFloat64 - pub const SME_B16B16: Caps = Caps::bit(92); // SME BFloat16xBFloat16 (Apple M5) - pub const SME_F16F16: Caps = Caps::bit(93); // SME Float16xFloat16 (Apple M5) - pub const SME_I8I32: Caps = Caps::bit(103); // SME Int8xInt32 - pub const SME_F16F32: Caps = Caps::bit(104); // SME Float16xFloat32 - pub const SME_B16F32: Caps = Caps::bit(105); // SME BFloat16xFloat32 - pub const SME_F32F32: Caps = Caps::bit(106); // SME Float32xFloat32 - pub const SME_FA64: Caps = Caps::bit(107); // SME Full A64 - pub const SME_I16I32: Caps = Caps::bit(108); // SME Int16xInt32 - pub const SME_BI32I32: Caps = Caps::bit(109); // SME BrainInt32xInt32 - pub const EBF16: Caps = Caps::bit(110); // Extended BFloat16 (NEON) + /// Scalable Matrix Extension version 2.1. + pub const SME2P1: Caps = Caps::bit(89); + /// SME 16-bit integer outer products accumulated into 64-bit elements. + pub const SME_I16I64: Caps = Caps::bit(90); + /// SME double-precision floating-point outer products. + pub const SME_F64F64: Caps = Caps::bit(91); + /// SME BFloat16 outer products accumulated into BFloat16 elements. + pub const SME_B16B16: Caps = Caps::bit(92); + /// SME half-precision outer products accumulated into half-precision elements. + pub const SME_F16F16: Caps = Caps::bit(93); + /// SME 8-bit integer outer products accumulated into 32-bit elements. + pub const SME_I8I32: Caps = Caps::bit(103); + /// SME half-precision outer products accumulated into single-precision elements. + pub const SME_F16F32: Caps = Caps::bit(104); + /// SME BFloat16 outer products accumulated into single-precision elements. + pub const SME_B16F32: Caps = Caps::bit(105); + /// SME single-precision floating-point outer products. + pub const SME_F32F32: Caps = Caps::bit(106); + /// Full A64 instruction availability in SME streaming mode. + pub const SME_FA64: Caps = Caps::bit(107); + /// SME 16-bit integer outer products accumulated into 32-bit elements. + pub const SME_I16I32: Caps = Caps::bit(108); + /// SME matrix operations selected by the architectural BI32I32 field. + pub const SME_BI32I32: Caps = Caps::bit(109); + /// Advanced SIMD extended BFloat16 instructions. + pub const EBF16: Caps = Caps::bit(110); // ─── SVE2.1 features ─── - pub const SVE2P1: Caps = Caps::bit(94); // SVE version 2.1 - pub const SVE_B16B16: Caps = Caps::bit(95); // SVE BFloat16xBFloat16 + /// Scalable Vector Extension version 2.1. + pub const SVE2P1: Caps = Caps::bit(94); + /// SVE BFloat16 multiply-accumulate instructions with BFloat16 results. + pub const SVE_B16B16: Caps = Caps::bit(95); // ─── Hardware RNG ─── - pub const RNG: Caps = Caps::bit(96); // RNDR/RNDRRS + /// Random-number instructions RNDR and RNDRRS. + pub const RNG: Caps = Caps::bit(96); // ─── Combined Capability Masks ─── @@ -563,45 +673,78 @@ pub mod riscv { use super::Caps; // ─── Vector Extension ─── + /// Standard vector extension. pub const V: Caps = Caps::bit(128); // ─── Bit Manipulation ─── + /// Basic bit-manipulation instructions. pub const ZBB: Caps = Caps::bit(129); + /// Single-bit manipulation instructions. pub const ZBS: Caps = Caps::bit(130); + /// Address-generation instructions. pub const ZBA: Caps = Caps::bit(131); - pub const ZBC: Caps = Caps::bit(132); // Carryless multiply + /// Carry-less multiplication instructions. + pub const ZBC: Caps = Caps::bit(132); // ─── Scalar Crypto ─── + /// Bit-manipulation instructions for cryptography. pub const ZBKB: Caps = Caps::bit(133); + /// Carry-less multiplication instructions for cryptography. pub const ZBKC: Caps = Caps::bit(134); + /// Crossbar-permutation instructions for cryptography. pub const ZBKX: Caps = Caps::bit(135); - pub const ZKND: Caps = Caps::bit(136); // AES decrypt - pub const ZKNE: Caps = Caps::bit(137); // AES encrypt - pub const ZKNH: Caps = Caps::bit(138); // SHA2 - pub const ZKSED: Caps = Caps::bit(139); // SM4 - pub const ZKSH: Caps = Caps::bit(140); // SM3 - pub const ZKN: Caps = Caps::bit(150); // NIST crypto bundle - pub const ZKS: Caps = Caps::bit(151); // ShangMi crypto bundle - pub const ZK: Caps = Caps::bit(152); // Standard scalar crypto bundle - pub const ZKT: Caps = Caps::bit(153); // Data-independent execution latency + /// NIST-suite AES decryption and key-schedule instructions. + pub const ZKND: Caps = Caps::bit(136); + /// NIST-suite AES encryption and key-schedule instructions. + pub const ZKNE: Caps = Caps::bit(137); + /// NIST-suite SHA-2 hash instructions. + pub const ZKNH: Caps = Caps::bit(138); + /// ShangMi-suite SM4 block-cipher instructions. + pub const ZKSED: Caps = Caps::bit(139); + /// ShangMi-suite SM3 hash instructions. + pub const ZKSH: Caps = Caps::bit(140); + /// Scalar NIST cryptography suite. + pub const ZKN: Caps = Caps::bit(150); + /// Scalar ShangMi cryptography suite. + pub const ZKS: Caps = Caps::bit(151); + /// Standard scalar cryptography suite. + pub const ZK: Caps = Caps::bit(152); + /// Scalar data-independent execution-latency guarantee. + pub const ZKT: Caps = Caps::bit(153); // ─── Vector Crypto ─── + /// Vector basic bit-manipulation instructions. pub const ZVBB: Caps = Caps::bit(141); + /// Vector carry-less multiplication instructions. pub const ZVBC: Caps = Caps::bit(142); + /// Vector bit-manipulation instructions for cryptography. pub const ZVKB: Caps = Caps::bit(143); + /// Vector GCM and GMAC instructions. pub const ZVKG: Caps = Caps::bit(144); + /// Vector AES block-cipher instructions. pub const ZVKNED: Caps = Caps::bit(145); + /// Vector SHA-256 hash instructions. pub const ZVKNHA: Caps = Caps::bit(146); + /// Vector SHA-256 and SHA-512 hash instructions. pub const ZVKNHB: Caps = Caps::bit(147); + /// Vector SM4 block-cipher instructions. pub const ZVKSED: Caps = Caps::bit(148); + /// Vector SM3 hash instructions. pub const ZVKSH: Caps = Caps::bit(149); - pub const ZVKT: Caps = Caps::bit(154); // Vector data-independent execution latency - pub const ZVKN: Caps = Caps::bit(155); // Vector NIST crypto bundle - pub const ZVKNC: Caps = Caps::bit(156); // Vector NIST crypto bundle + CLMUL - pub const ZVKNG: Caps = Caps::bit(157); // Vector NIST crypto bundle + GCM - pub const ZVKS: Caps = Caps::bit(158); // Vector ShangMi crypto bundle - pub const ZVKSC: Caps = Caps::bit(159); // Vector ShangMi crypto bundle + CLMUL - pub const ZVKSG: Caps = Caps::bit(160); // Vector ShangMi crypto bundle + GCM + /// Vector data-independent execution-latency guarantee. + pub const ZVKT: Caps = Caps::bit(154); + /// Vector NIST cryptography suite. + pub const ZVKN: Caps = Caps::bit(155); + /// Vector NIST cryptography suite with carry-less multiplication. + pub const ZVKNC: Caps = Caps::bit(156); + /// Vector NIST cryptography suite with GCM and GMAC instructions. + pub const ZVKNG: Caps = Caps::bit(157); + /// Vector ShangMi cryptography suite. + pub const ZVKS: Caps = Caps::bit(158); + /// Vector ShangMi cryptography suite with carry-less multiplication. + pub const ZVKSC: Caps = Caps::bit(159); + /// Vector ShangMi cryptography suite with GCM and GMAC instructions. + pub const ZVKSG: Caps = Caps::bit(160); } // WebAssembly Features (bits 192-207) @@ -610,7 +753,9 @@ pub mod riscv { pub mod wasm { use super::Caps; + /// WebAssembly 128-bit SIMD instructions. pub const SIMD128: Caps = Caps::bit(192); + /// WebAssembly relaxed SIMD instructions. pub const RELAXED_SIMD: Caps = Caps::bit(193); } diff --git a/src/platform/detect.rs b/src/platform/detect.rs index 278c3dec..232e84ed 100644 --- a/src/platform/detect.rs +++ b/src/platform/detect.rs @@ -1,23 +1,3 @@ -//! CPU feature detection. -//! -//! This module provides runtime CPU capability detection with caching. -//! It handles compile-time and runtime detection, caching, and user overrides. -//! -//! # Detection Tiers -//! -//! 1. **Compile-time**: `cfg!(target_feature)` - zero cost, dead code elimination -//! 2. **Runtime (std)**: `is_x86_feature_detected!` + `OnceLock` caching -//! 3. **Runtime (no_std)**: Atomic-based caching for embedded targets -//! -//! # Override Support -//! -//! ``` -//! use rscrypto::platform::{Detected, expert}; -//! expert::try_set_override(Some(Detected::portable()))?; -//! expert::try_set_override(None)?; -//! # Ok::<(), rscrypto::platform::expert::OverrideError>(()) -//! ``` - use crate::platform::caps::{Arch, Caps}; /// Errors when configuring runtime detection overrides. @@ -120,15 +100,9 @@ fn validate_override(value: Option) -> Result, Overri /// /// Results are cached after first call. /// -/// # Examples -/// -/// ``` -/// let det = rscrypto::platform::get(); -/// assert_eq!(det.arch, rscrypto::platform::Arch::current()); -/// ``` #[inline] #[must_use] -pub fn get() -> Detected { +pub(super) fn get() -> Detected { // Miri cannot interpret SIMD intrinsics #[cfg(miri)] { @@ -137,8 +111,7 @@ pub fn get() -> Detected { #[cfg(not(miri))] { - #[allow(unused_mut)] - let mut det = { + let det = { #[cfg(feature = "std")] { *STD_CACHE.get_or_init(detect_with_override) @@ -172,7 +145,7 @@ pub fn get() -> Detected { /// `Cargo.toml` for deployment context (FIPS / DO-178C / ISO 26262). #[inline] #[must_use] -pub fn caps() -> Caps { +pub(super) fn caps() -> Caps { #[cfg(feature = "portable-only")] { Caps::NONE @@ -186,7 +159,7 @@ pub fn caps() -> Caps { /// Get the detected architecture. #[inline] #[must_use] -pub fn arch() -> Arch { +pub(super) fn arch() -> Arch { get().arch } diff --git a/src/platform/detect/arch/aarch64.rs b/src/platform/detect/arch/aarch64.rs index c3011446..b929e8ab 100644 --- a/src/platform/detect/arch/aarch64.rs +++ b/src/platform/detect/arch/aarch64.rs @@ -380,6 +380,7 @@ fn runtime_aarch64() -> Caps { #[cfg(all( target_arch = "aarch64", feature = "std", + any(test, feature = "crc16", feature = "crc24", feature = "crc32", feature = "crc64"), any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos") ))] #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -403,7 +404,6 @@ enum AppleSiliconGen { feature = "std", any(feature = "crc16", feature = "crc24", feature = "crc32", feature = "crc64") ))] -#[cfg_attr(miri, allow(dead_code))] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Aarch64TuneFamily { #[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos"))] @@ -427,6 +427,7 @@ pub(crate) enum Aarch64TuneFamily { #[cfg(all( target_arch = "aarch64", feature = "std", + any(test, feature = "crc16", feature = "crc24", feature = "crc32", feature = "crc64"), any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos") ))] fn detect_apple_silicon_gen() -> Option { @@ -459,9 +460,7 @@ fn detect_apple_silicon_gen() -> Option { // Direct extern "C" linkage to libSystem's sysctlbyname // (libSystem is always linked on Apple platforms) - // SAFETY: This extern block declares a C function from libSystem. // The function signature matches Apple's sysctlbyname(3). - #[allow(unsafe_code)] unsafe extern "C" { fn sysctlbyname( name: *const u8, @@ -478,7 +477,6 @@ fn detect_apple_silicon_gen() -> Option { // SAFETY: sysctlbyname is safe to call with valid pointers. // "hw.cpufamily" is a valid null-terminated string. // The output buffer is properly sized for u32. - #[allow(unsafe_code)] let ret = unsafe { sysctlbyname( c"hw.cpufamily".as_ptr().cast(), @@ -500,10 +498,10 @@ fn detect_apple_silicon_gen() -> Option { CPUFAMILY_ARM_DONAN | CPUFAMILY_ARM_BRAVA => Some(AppleSiliconGen::M4), CPUFAMILY_ARM_HIDRA | CPUFAMILY_ARM_SOTRA => Some(AppleSiliconGen::M5), // A-series chips - treat as M-series equivalent for tuning - CPUFAMILY_ARM_COLL => Some(AppleSiliconGen::M2), // A17 Pro ≈ M2 architecture + CPUFAMILY_ARM_COLL => Some(AppleSiliconGen::M2), // A17 Pro ≈ M2 architecture CPUFAMILY_ARM_TAHITI | CPUFAMILY_ARM_TUPAI => Some(AppleSiliconGen::M4), // A18 ≈ M4 architecture - CPUFAMILY_ARM_TILOS | CPUFAMILY_ARM_THERA => Some(AppleSiliconGen::M5), // A19 ≈ M5 architecture - _ => None, // Unknown future chip - will fall back to feature-based detection + CPUFAMILY_ARM_TILOS | CPUFAMILY_ARM_THERA => Some(AppleSiliconGen::M5), // A19 ≈ M5 architecture + _ => None, // Unknown future chip - will fall back to feature-based detection } } @@ -514,7 +512,6 @@ fn detect_apple_silicon_gen() -> Option { any(feature = "crc16", feature = "crc24", feature = "crc32", feature = "crc64") ))] #[must_use] -#[cfg_attr(miri, allow(dead_code))] pub(crate) fn detect_aarch64_tune_family() -> Option { #[cfg(all( feature = "std", @@ -552,8 +549,6 @@ fn detect_apple_sme_features() -> Caps { // Helper to read a u32 sysctl value (returns 0 on error or false, 1 on true) fn sysctl_u32(name: &[u8]) -> u32 { // Direct extern "C" linkage to libSystem's sysctlbyname - // SAFETY: This extern block declares a C function from libSystem. - #[allow(unsafe_code)] unsafe extern "C" { fn sysctlbyname( name: *const u8, @@ -570,7 +565,6 @@ fn detect_apple_sme_features() -> Caps { // SAFETY: sysctlbyname is safe to call with valid pointers. // name is a valid null-terminated C string. // The output buffer is properly sized for u32. - #[allow(unsafe_code)] let ret = unsafe { sysctlbyname( name.as_ptr(), @@ -614,171 +608,6 @@ fn detect_apple_sme_features() -> Caps { caps } -// TODO(T7): Feed Apple SVL into production SME dispatch only after an -// SVL-sensitive kernel wins representative target-native benchmarks. -/// Detect SME streaming vector length in bytes on Apple platforms. -/// -/// Returns the maximum SVL in bytes, or 0 if SME is not supported or detection failed. -/// -/// On Apple Silicon: -/// - M4: SME with 128-bit tiles (SVL = 16 bytes) -/// - M5: SME2p1 with 128-bit tiles (SVL = 16 bytes) -/// -/// Note: Apple's implementation uses fixed 128-bit SVL, unlike server ARM chips -/// which may support 128-512 bit configurable SVL. -#[cfg(all( - target_arch = "aarch64", - feature = "std", - any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos") -))] -#[allow(dead_code)] -fn detect_apple_sme_tile_size() -> u16 { - fn sysctl_u32(name: &[u8]) -> u32 { - #[allow(unsafe_code)] - unsafe extern "C" { - fn sysctlbyname( - name: *const u8, - oldp: *mut core::ffi::c_void, - oldlenp: *mut usize, - newp: *const core::ffi::c_void, - newlen: usize, - ) -> i32; - } - - let mut value: u32 = 0; - let mut size = core::mem::size_of::(); - - #[allow(unsafe_code)] - // SAFETY: `sysctlbyname` expects `name` to be a valid NUL-terminated C string (caller provides this), - // `oldp`/`oldlenp` point to writable locals, and `newp` is null with `newlen = 0` (no write). - let ret = unsafe { - sysctlbyname( - name.as_ptr(), - core::ptr::addr_of_mut!(value).cast(), - core::ptr::addr_of_mut!(size), - core::ptr::null(), - 0, - ) - }; - - if ret == 0 { value } else { 0 } - } - - let svl_bytes = sysctl_u32(c"hw.optional.arm.sme_max_svl_b".to_bytes_with_nul()); - if svl_bytes > 0 { - return svl_bytes as u16; - } - - if let Some(AppleSiliconGen::M4 | AppleSiliconGen::M5) = detect_apple_silicon_gen() { - return 16; - } - - 0 -} - -// TODO(T7): Feed SVE VL into production dispatch only after a length-sensitive -// SVE kernel wins representative target-native benchmarks. -/// Detect SVE vector length in bits via prctl(PR_SVE_GET_VL). -/// -/// Uses raw syscall to avoid libc dependency. Returns 0 if SVE is not supported. -#[cfg(all(target_arch = "aarch64", target_os = "linux", feature = "std"))] -#[allow(dead_code)] -fn detect_sve_vlen() -> u16 { - const SYS_PRCTL: u64 = 167; - const PR_SVE_GET_VL: u64 = 51; - const PR_SVE_VL_LEN_MASK: u64 = 0xFFFF; - - let result: i64; - - // SAFETY: prctl(PR_SVE_GET_VL) is always safe to call. - // Returns the vector length in bytes on success, or -EINVAL if SVE unsupported. - #[allow(unsafe_code)] - unsafe { - core::arch::asm!( - "svc #0", - in("x8") SYS_PRCTL, - in("x0") PR_SVE_GET_VL, - in("x1") 0u64, - in("x2") 0u64, - in("x3") 0u64, - in("x4") 0u64, - lateout("x0") result, - options(nostack) - ); - } - - if result < 0 { - return 0; - } - - let vl_bytes = (result as u64) & PR_SVE_VL_LEN_MASK; - let vl_bits = vl_bytes.strict_mul(8); - if vl_bits > u16::MAX as u64 { - u16::MAX - } else { - vl_bits as u16 - } -} - -/// Fallback SVE vector length detection for non-Linux platforms. -#[cfg(all(target_arch = "aarch64", not(all(target_os = "linux", feature = "std"))))] -#[allow(dead_code)] -fn detect_sve_vlen() -> u16 { - 0 -} - -// TODO(T7): Feed SME VL into production dispatch only after a length-sensitive -// SME kernel wins representative target-native benchmarks. -/// Detect SME streaming vector length in bits via prctl(PR_SME_GET_VL). -/// -/// Uses raw syscall to avoid libc dependency. Returns 0 if SME is not supported. -/// The SME vector length determines the tile size (SVL × SVL bits). -#[cfg(all(target_arch = "aarch64", target_os = "linux", feature = "std"))] -#[allow(dead_code)] -fn detect_sme_vlen() -> u16 { - const SYS_PRCTL: u64 = 167; - const PR_SME_GET_VL: u64 = 63; - const PR_SME_VL_LEN_MASK: u64 = 0xFFFF; - - let result: i64; - - // SAFETY: prctl(PR_SME_GET_VL) is always safe to call. - // Returns the streaming vector length in bytes on success, or -EINVAL if SME unsupported. - #[allow(unsafe_code)] - unsafe { - core::arch::asm!( - "svc #0", - in("x8") SYS_PRCTL, - in("x0") PR_SME_GET_VL, - in("x1") 0u64, - in("x2") 0u64, - in("x3") 0u64, - in("x4") 0u64, - lateout("x0") result, - options(nostack) - ); - } - - if result < 0 { - return 0; - } - - let vl_bytes = (result as u64) & PR_SME_VL_LEN_MASK; - let vl_bits = vl_bytes.strict_mul(8); - if vl_bits > u16::MAX as u64 { - u16::MAX - } else { - vl_bits as u16 - } -} - -/// Fallback SME vector length detection for non-Linux platforms. -#[cfg(all(target_arch = "aarch64", not(all(target_os = "linux", feature = "std"))))] -#[allow(dead_code)] -fn detect_sme_vlen() -> u16 { - 0 -} - // MIDR_EL1 Detection (Linux aarch64) #[cfg(all( @@ -859,7 +688,10 @@ fn parse_u32_auto_radix(value: &str) -> Option { return u32::from_str_radix(hex, 16).ok(); } - value.parse::().ok().or_else(|| u32::from_str_radix(value, 16).ok()) + value + .parse::() + .ok() + .or_else(|| u32::from_str_radix(value, 16).ok()) } #[cfg(all( diff --git a/src/platform/detect/arch/power.rs b/src/platform/detect/arch/power.rs index 4e8c685f..3cf05471 100644 --- a/src/platform/detect/arch/power.rs +++ b/src/platform/detect/arch/power.rs @@ -40,9 +40,10 @@ fn runtime_power() -> Caps { let mut hwcap = 0u64; let mut hwcap2 = 0u64; - for chunk in buf.get(..n)?.chunks_exact(16) { - let a_type = u64::from_ne_bytes(chunk.get(0..8)?.try_into().ok()?); - let a_val = u64::from_ne_bytes(chunk.get(8..16)?.try_into().ok()?); + for chunk in buf.get(..n)?.as_chunks::<16>().0 { + let (types, values) = chunk.as_chunks::<8>().0.split_at(1); + let a_type = u64::from_ne_bytes(types[0]); + let a_val = u64::from_ne_bytes(values[0]); if a_type == AT_HWCAP { hwcap = a_val; diff --git a/src/platform/detect/arch/riscv.rs b/src/platform/detect/arch/riscv.rs index e5f9541a..7b421512 100644 --- a/src/platform/detect/arch/riscv.rs +++ b/src/platform/detect/arch/riscv.rs @@ -72,6 +72,12 @@ fn runtime_riscv() -> Caps { const RISCV_HWPROBE_EXT_ZVKT: u64 = 1 << 26; #[inline] + /// Invoke the Linux RISC-V hardware-probe syscall. + /// + /// # Safety + /// + /// `pairs` must be valid for writes to `pair_count` consecutive `RiscvHwprobe` + /// values and must remain valid until the syscall returns. unsafe fn syscall_riscv_hwprobe(pairs: *mut RiscvHwprobe, pair_count: usize) -> isize { let ret: usize; // SAFETY: We follow the Linux RISC-V syscall ABI directly: @@ -89,7 +95,7 @@ fn runtime_riscv() -> Caps { options(nostack, preserves_flags), ); } - ret as isize + isize::from_ne_bytes(ret.to_ne_bytes()) } let mut probes = [ diff --git a/src/platform/detect/arch/s390x.rs b/src/platform/detect/arch/s390x.rs index e575385a..85cf8dd1 100644 --- a/src/platform/detect/arch/s390x.rs +++ b/src/platform/detect/arch/s390x.rs @@ -47,9 +47,10 @@ fn runtime_s390x_linux() -> Caps { let mut buf = [0u8; 4096]; let n = file.read(&mut buf).ok()?; - for chunk in buf.get(..n)?.chunks_exact(16) { - let a_type = u64::from_ne_bytes(chunk.get(0..8)?.try_into().ok()?); - let a_val = u64::from_ne_bytes(chunk.get(8..16)?.try_into().ok()?); + for chunk in buf.get(..n)?.as_chunks::<16>().0 { + let (types, values) = chunk.as_chunks::<8>().0.split_at(1); + let a_type = u64::from_ne_bytes(types[0]); + let a_val = u64::from_ne_bytes(values[0]); if a_type == AT_HWCAP { return Some(a_val); } @@ -64,12 +65,12 @@ fn runtime_s390x_linux() -> Caps { let stfle = stfle_facilities(); #[inline(always)] - fn has_facility(words: &[u64; 4], bit: usize) -> bool { - // SAFETY: Facility bits are 0-255; words[bit/64] accesses indices 0-3 in a 4-element array. - // Using get() satisfies clippy::indexing_slicing while maintaining performance. + fn has_facility(words: &[u64; 4], bit: u8) -> bool { + let word_index = usize::from(bit >> 6); + let shift = 63u32.strict_sub(u32::from(bit & 63)); words - .get(bit / 64) - .map(|w| w & (1u64 << (63 - (bit % 64))) != 0) + .get(word_index) + .map(|word| word & 1u64.strict_shl(shift) != 0) .unwrap_or(false) } @@ -152,7 +153,7 @@ fn stfle_facilities() -> [u64; 4] { core::arch::asm!( "stfle 0({ptr})", ptr = in(reg) facilities.as_mut_ptr(), - inout("r0") facilities.len() as u64 - 1 => _, + inout("r0") 3u64 => _, options(nostack) ); } diff --git a/src/platform/detect/arch/x86.rs b/src/platform/detect/arch/x86.rs index 17835c4e..10b49387 100644 --- a/src/platform/detect/arch/x86.rs +++ b/src/platform/detect/arch/x86.rs @@ -20,8 +20,16 @@ fn detect_x86_64() -> Detected { #[cfg(feature = "std")] let mut caps = caps_static.union(runtime_caps); - #[cfg(not(feature = "std"))] + #[cfg(all( + not(feature = "std"), + any(target_os = "linux", target_os = "android") + ))] let mut caps = caps_static; + #[cfg(all( + not(feature = "std"), + not(any(target_os = "linux", target_os = "android")) + ))] + let caps = caps_static; #[cfg(feature = "std")] { @@ -36,7 +44,7 @@ fn detect_x86_64() -> Detected { caps = gate_x86_amx_permission(caps, false); } - // Hybrid Intel AVX-512 Safety: Clear AVX-512 caps on hybrid CPUs + // Clear AVX-512 capabilities on hybrid Intel CPUs. // On hybrid Intel CPUs (Alder Lake, Raptor Lake, etc.), the P-cores have // AVX-512 but E-cores don't. If a thread migrates to an E-core while // executing AVX-512 code, it will SIGILL. The only safe approach is to @@ -74,14 +82,20 @@ fn detect_x86_64() -> Detected { } } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "std", target_os = "linux", target_os = "android") +))] const X86_ALL_AMX: Caps = crate::platform::caps::x86::AMX_TILE .union(crate::platform::caps::x86::AMX_BF16) .union(crate::platform::caps::x86::AMX_INT8) .union(crate::platform::caps::x86::AMX_FP16) .union(crate::platform::caps::x86::AMX_COMPLEX); -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "std", target_os = "linux", target_os = "android") +))] #[inline] const fn gate_x86_amx_permission(caps: Caps, permitted: bool) -> Caps { if permitted { @@ -115,32 +129,41 @@ fn detect_x86() -> Detected { #[cfg(all(target_arch = "x86_64", feature = "std"))] #[inline] -// MSRV: CPUID is unsafe on Rust 1.91 but safe on the pinned nightly. -#[allow(unsafe_code, unused_unsafe)] -fn cpuid_leaf(leaf: u32) -> core::arch::x86_64::CpuidResult { - // SAFETY: CPUID leaf read is safe here because: - // 1. This function is compiled only for x86_64 targets. - // 2. CPUID is a non-privileged CPU-identification instruction. - // 3. The intrinsic returns register values and does not access Rust memory. - unsafe { core::arch::x86_64::__cpuid(leaf) } +fn cpuid_leaf(leaf: u32) -> CpuidRegisters { + cpuid_leaf_count(leaf, 0) } #[cfg(all(target_arch = "x86_64", feature = "std"))] #[inline] -// MSRV: CPUID is unsafe on Rust 1.91 but safe on the pinned nightly. -#[allow(unsafe_code, unused_unsafe)] -fn cpuid_leaf_count(leaf: u32, subleaf: u32) -> core::arch::x86_64::CpuidResult { - // SAFETY: CPUID leaf/subleaf read is safe here because: - // 1. This function is compiled only for x86_64 targets. - // 2. CPUID is a non-privileged CPU-identification instruction. - // 3. The intrinsic returns register values and does not access Rust memory. - unsafe { core::arch::x86_64::__cpuid_count(leaf, subleaf) } +fn cpuid_leaf_count(leaf: u32, subleaf: u32) -> CpuidRegisters { + let mut eax = leaf; + let mut ecx = subleaf; + let ebx: u32; + let edx: u32; + + // SAFETY: CPUID is a non-privileged identification instruction on x86_64. + // The sequence preserves RBX, reports every modified register, restores RSP, + // and neither reads nor writes Rust memory. + unsafe { + core::arch::asm!( + "push rbx", + "cpuid", + "mov {ebx:e}, ebx", + "pop rbx", + ebx = lateout(reg) ebx, + inout("eax") eax, + inout("ecx") ecx, + lateout("edx") edx, + options(preserves_flags), + ); + } + + CpuidRegisters { eax, ebx, ecx, edx } } #[cfg(all(target_arch = "x86", feature = "std"))] #[inline] // MSRV: CPUID is unsafe on Rust 1.91 but safe on the pinned nightly. -#[allow(unsafe_code, unused_unsafe)] fn cpuid_leaf(leaf: u32) -> core::arch::x86::CpuidResult { // SAFETY: CPUID leaf read is safe here because: // 1. This function is compiled only for x86 targets. @@ -174,18 +197,6 @@ struct CpuidRegisters { edx: u32, } -#[cfg(all(target_arch = "x86_64", feature = "std"))] -impl From for CpuidRegisters { - fn from(result: core::arch::x86_64::CpuidResult) -> Self { - Self { - eax: result.eax, - ebx: result.ebx, - ecx: result.ecx, - edx: result.edx, - } - } -} - #[cfg(all(target_arch = "x86_64", feature = "std"))] #[derive(Clone, Copy, Default)] struct CpuidSnapshot { @@ -214,37 +225,35 @@ struct CpuidSnapshot { /// OSXSAVE and XGETBV(XCR0) to ensure the OS will save/restore extended registers. /// Without this check, using AVX/AVX-512 instructions could cause SIGILL. /// -/// # Safety -/// Uses XGETBV, which requires `unsafe` and is only called when OSXSAVE is set. +/// XGETBV is read only after CPUID reports OSXSAVE support. #[cfg(all(target_arch = "x86_64", feature = "std"))] -#[allow(unsafe_code)] fn cpuid_batch_x86_64() -> CpuidBatch { use core::arch::x86_64::_xgetbv; - let leaf0 = CpuidRegisters::from(cpuid_leaf(0)); + let leaf0 = cpuid_leaf(0); let leaf1 = if leaf0.eax >= 1 { - CpuidRegisters::from(cpuid_leaf(1)) + cpuid_leaf(1) } else { CpuidRegisters::default() }; let leaf7_0 = if leaf0.eax >= 7 { - CpuidRegisters::from(cpuid_leaf_count(7, 0)) + cpuid_leaf_count(7, 0) } else { CpuidRegisters::default() }; let leaf7_1 = if leaf0.eax >= 7 && leaf7_0.eax >= 1 { - CpuidRegisters::from(cpuid_leaf_count(7, 1)) + cpuid_leaf_count(7, 1) } else { CpuidRegisters::default() }; let leaf24_0 = if leaf0.eax >= 0x24 && leaf7_1.edx & (1 << 19) != 0 { - CpuidRegisters::from(cpuid_leaf_count(0x24, 0)) + cpuid_leaf_count(0x24, 0) } else { CpuidRegisters::default() }; - let extended_leaf0 = CpuidRegisters::from(cpuid_leaf(0x8000_0000)); + let extended_leaf0 = cpuid_leaf(0x8000_0000); let extended_leaf1 = if extended_leaf0.eax >= 0x8000_0001 { - CpuidRegisters::from(cpuid_leaf(0x8000_0001)) + cpuid_leaf(0x8000_0001) } else { CpuidRegisters::default() }; @@ -274,7 +283,6 @@ fn cpuid_batch_x86_64() -> CpuidBatch { feature = "std", any(target_os = "linux", target_os = "android") ))] -#[allow(unsafe_code)] fn amx_xstate_permission_x86_64() -> bool { const SYS_ARCH_PRCTL: isize = 158; const ARCH_GET_XCOMP_PERM: usize = 0x1022; @@ -294,7 +302,7 @@ fn amx_xstate_permission_x86_64() -> bool { "syscall", inlateout("rax") result, in("rdi") ARCH_GET_XCOMP_PERM, - in("rsi") &mut permissions, + in("rsi") &raw mut permissions, lateout("rcx") _, lateout("r11") _, options(nostack), @@ -358,13 +366,17 @@ fn decode_cpuid_x86_64(snapshot: CpuidSnapshot) -> CpuidBatch { // Extract extended family (bits 27:20) + base family (bits 11:8) let base_family = (cpuid1.eax >> 8) & 0xF; let ext_family = (cpuid1.eax >> 20) & 0xFF; - let family = base_family + ext_family; + let family = if base_family == 0xF { + base_family.strict_add(ext_family) + } else { + base_family + }; // Extract model (bits 7:4 + extended model bits 19:16 for family 6/15) let base_model = (cpuid1.eax >> 4) & 0xF; let ext_model = (cpuid1.eax >> 16) & 0xF; let model = if base_family == 6 || base_family == 15 { - base_model + (ext_model << 4) + base_model | ext_model.strict_shl(4) } else { base_model }; @@ -596,7 +608,6 @@ fn decode_cpuid_x86_64(snapshot: CpuidSnapshot) -> CpuidBatch { /// # Safety /// Uses CPUID instruction which requires unsafe, but is always safe to call on x86. #[cfg(all(target_arch = "x86", feature = "std"))] -#[allow(unsafe_code)] fn runtime_x86_32() -> Caps { use crate::platform::caps::x86; diff --git a/src/platform/detect/cache_override.rs b/src/platform/detect/cache_override.rs index ec083fc1..4f62bb30 100644 --- a/src/platform/detect/cache_override.rs +++ b/src/platform/detect/cache_override.rs @@ -140,7 +140,7 @@ mod atomic_cache { } #[cfg(all(not(feature = "std"), target_has_atomic = "64"))] - pub fn get_or_init(f: fn() -> Detected) -> Detected { + pub(super) fn get_or_init(f: fn() -> Detected) -> Detected { if STATE.load(Ordering::Acquire) == STATE_READY { return load_cached(); } @@ -176,7 +176,7 @@ mod atomic_cache { } } - pub fn try_set_override(value: Option, already_initialized: bool) -> Result<(), OverrideError> { + pub(super) fn try_set_override(value: Option, already_initialized: bool) -> Result<(), OverrideError> { let _guard = lock_override(); if OVERRIDE_SEALED.load(Ordering::Relaxed) { @@ -204,12 +204,12 @@ mod atomic_cache { Ok(()) } - pub fn has_override() -> bool { + pub(super) fn has_override() -> bool { OVERRIDE_SET.load(Ordering::Acquire) } #[cfg(not(miri))] - pub fn seal_and_get_override() -> Option { + pub(super) fn seal_and_get_override() -> Option { let _guard = lock_override(); OVERRIDE_SEALED.store(true, Ordering::Relaxed); diff --git a/src/platform/detect/compile_time.rs b/src/platform/detect/compile_time.rs index ce430373..cb70f02b 100644 --- a/src/platform/detect/compile_time.rs +++ b/src/platform/detect/compile_time.rs @@ -5,73 +5,62 @@ /// Detects features enabled via `-C target-feature=...` or `-C target-cpu=native`. /// Returns a `const` value—the compiler eliminates all runtime checks. /// -/// # When to Use -/// -/// - Building specialized binaries for known hardware -/// - Maximum performance when target features are guaranteed -/// - Embedded/bare-metal where runtime detection isn't available -/// -/// For generic binaries that run on multiple CPUs, use [`get()`] instead. -/// -/// # Examples -/// -/// ``` -/// use rscrypto::platform::caps_static; -/// -/// // Evaluates at compile time—no runtime cost -/// const CAPS: rscrypto::platform::Caps = caps_static(); -/// -/// // On x86_64, SSE2 is always present -/// #[cfg(target_arch = "x86_64")] -/// { -/// use rscrypto::platform::caps::x86; -/// assert!(CAPS.has(x86::SSE2)); -/// } -/// -/// // On aarch64, NEON is always present -/// #[cfg(target_arch = "aarch64")] -/// { -/// use rscrypto::platform::caps::aarch64; -/// assert!(CAPS.has(aarch64::NEON)); -/// } -/// ``` -/// -/// When compiled with `-C target-cpu=znver4`: -/// -/// ``` -/// # #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] -/// # fn example() { -/// use rscrypto::platform::{caps::x86, caps_static}; -/// -/// const CAPS: rscrypto::platform::Caps = caps_static(); -/// // AVX-512 features are detected at compile time -/// assert!(CAPS.has(x86::AVX512F)); -/// # } -/// ``` -/// /// # Implementation /// /// Uses `cfg!()` macro inside `const fn`. The compiler evaluates `cfg!()` at /// compile time and eliminates dead branches via constant propagation. #[inline(always)] #[must_use] -pub const fn caps_static() -> Caps { +pub(super) const fn caps_static() -> Caps { // Note: imports are architecture-conditional to avoid warnings use crate::platform::caps::Caps; // Declarative macro for compile-time feature detection. // Uses cfg!() which returns a const bool, enabling use in const fn. // The compiler eliminates dead branches entirely. - #[allow(unused_macros)] // Only used on x86/x86_64/aarch64 + #[cfg(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "riscv32", + target_arch = "wasm32", + target_arch = "wasm64", + target_arch = "s390x", + target_arch = "powerpc64" + ))] macro_rules! detect { ($caps:ident; $($feature:literal => $cap:expr),+ $(,)?) => { $(if cfg!(target_feature = $feature) { $caps = $caps.union($cap); })+ }; } - #[allow(unused_mut)] + #[cfg(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "riscv32", + target_arch = "wasm32", + target_arch = "wasm64", + target_arch = "s390x", + target_arch = "powerpc64" + ))] let mut result = Caps::NONE; + #[cfg(not(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "riscv32", + target_arch = "wasm32", + target_arch = "wasm64", + target_arch = "s390x", + target_arch = "powerpc64" + )))] + return Caps::NONE; + // x86/x86_64 #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { @@ -305,5 +294,16 @@ pub const fn caps_static() -> Caps { ); } + #[cfg(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "riscv32", + target_arch = "wasm32", + target_arch = "wasm64", + target_arch = "s390x", + target_arch = "powerpc64" + ))] result } diff --git a/src/platform/detect/tests.rs b/src/platform/detect/tests.rs index 4659d670..954c4cab 100644 --- a/src/platform/detect/tests.rs +++ b/src/platform/detect/tests.rs @@ -258,21 +258,6 @@ mod tests { } } - // SVE Vector Length Detection Tests - - #[test] - #[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri)))] - fn test_sve_vlen_detection_runs() { - // Just verify detection doesn't crash - let vlen = detect_sve_vlen(); - // VL should be 0 (no SVE) or a valid power-of-2 in [128, 2048] - if vlen > 0 { - assert!(vlen >= 128, "SVE VL too small: {vlen}"); - assert!(vlen <= 2048, "SVE VL too large: {vlen}"); - assert!(vlen.is_power_of_two(), "SVE VL not power of 2: {vlen}"); - } - } - // Hybrid Intel Detection Tests #[test] @@ -616,18 +601,9 @@ mod tests { any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos") ))] fn test_macos_extended_features() { - // Test that new feature detection works on macOS use crate::platform::caps::aarch64; let det = get(); - // Verify extended features are detected on capable hardware - // On M1+, we should detect these features: - std::eprintln!("Detected features: {}", det.caps.count()); - std::eprintln!(" I8MM: {}", det.caps.has(aarch64::I8MM)); - std::eprintln!(" BF16: {}", det.caps.has(aarch64::BF16)); - std::eprintln!(" FRINTTS: {}", det.caps.has(aarch64::FRINTTS)); - std::eprintln!(" LSE2: {}", det.caps.has(aarch64::LSE2)); - // FRINTTS is detectable via std::arch on macOS; LSE2 is not exposed by // Apple's sysctl and therefore cannot be asserted here. assert!(det.caps.has(aarch64::FRINTTS), "FRINTTS should be detected on M1+"); @@ -640,54 +616,18 @@ mod tests { not(miri), any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos") ))] - fn test_detect_apple_sme_features_exists() { - // Verify the SME detection function exists and returns valid caps - let sme_caps = detect_apple_sme_features(); - // The function should always return valid Caps (may be empty on M1-M3) - // On M4+, SME should be detected - std::eprintln!("SME caps detected: {}", sme_caps.count()); - std::eprintln!(" SME: {}", sme_caps.has(crate::platform::caps::aarch64::SME)); - std::eprintln!(" SME2: {}", sme_caps.has(crate::platform::caps::aarch64::SME2)); - } - - #[test] - #[cfg(all( - target_arch = "aarch64", - feature = "std", - not(miri), - any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos") - ))] - fn test_detect_apple_silicon_gen_exists() { - // Verify chip generation detection works - if let Some(chip_gen) = detect_apple_silicon_gen() { - std::eprintln!("Detected Apple Silicon generation: {:?}", chip_gen); - // Basic sanity checks - match chip_gen { - AppleSiliconGen::M1 | AppleSiliconGen::M2 | AppleSiliconGen::M3 => { - // M1-M3 should not have SME - std::eprintln!("M1-M3 chip detected (no SME expected)"); - } - AppleSiliconGen::M4 => { - // M4 should have SME - std::eprintln!("M4 chip detected (SME expected)"); - } - AppleSiliconGen::M5 => { - // M5 should have SME2 - std::eprintln!("M5 chip detected (SME2 expected)"); - } - } - } else { - std::eprintln!("Unknown or A-series chip detected"); - } - } - - // Override Mechanism Tests + fn test_detect_apple_sme_features_are_scoped() { + use crate::platform::caps::aarch64; - #[test] - fn test_has_override_exists() { - // Verify the override API exists and returns a bool. - // Note: Due to global state from other tests, we can't assert a specific value. - let _ = has_override(); + let sme_caps = detect_apple_sme_features(); + let owned_caps = aarch64::SME + .union(aarch64::SME2) + .union(aarch64::SME2P1) + .union(aarch64::SME_I16I64) + .union(aarch64::SME_F64F64) + .union(aarch64::SME_B16B16) + .union(aarch64::SME_F16F16); + assert!(sme_caps.difference(owned_caps).is_empty()); } #[test] diff --git a/src/platform/mod.rs b/src/platform/mod.rs index bdef3c2d..1b0aa646 100644 --- a/src/platform/mod.rs +++ b/src/platform/mod.rs @@ -98,6 +98,14 @@ pub fn arch() -> Arch { /// /// Normal callers should use [`get()`], [`caps()`], or [`arch()`]. Overrides /// must be configured before the first cached detection. +/// +/// ``` +/// use rscrypto::platform::{Detected, expert}; +/// +/// expert::try_set_override(Some(Detected::portable()))?; +/// expert::try_set_override(None)?; +/// # Ok::<(), rscrypto::platform::expert::OverrideError>(()) +/// ``` pub mod expert { pub use super::detect::{OverrideError, detect_uncached, has_override, try_set_override}; } @@ -108,6 +116,27 @@ pub mod expert { /// or `-C target-cpu=native`. Use this for zero-overhead dispatch. /// /// This is a compile-time constant and performs no runtime detection. +/// For generic binaries that run on multiple CPUs, use [`caps()`] instead. +/// +/// # Examples +/// +/// ``` +/// use rscrypto::platform::caps_static; +/// +/// const CAPS: rscrypto::platform::Caps = caps_static(); +/// +/// #[cfg(target_arch = "x86_64")] +/// { +/// use rscrypto::platform::caps::x86; +/// assert!(CAPS.has(x86::SSE2)); +/// } +/// +/// #[cfg(target_arch = "aarch64")] +/// { +/// use rscrypto::platform::caps::aarch64; +/// assert!(CAPS.has(aarch64::NEON)); +/// } +/// ``` #[inline(always)] #[must_use] pub const fn caps_static() -> Caps { diff --git a/src/platform/target_matrix.rs b/src/platform/target_matrix.rs index 687c8901..07bbbfae 100644 --- a/src/platform/target_matrix.rs +++ b/src/platform/target_matrix.rs @@ -30,7 +30,7 @@ fn matches_target_arch(target: &str, arch: Arch) -> bool { /// an arch prefix like `x86_64-`, so false positives are impossible. #[inline] #[must_use] -pub fn manifest_has_arch(arch: Arch) -> bool { +pub(super) fn manifest_has_arch(arch: Arch) -> bool { let Some(_) = manifest_prefix_for_arch(arch) else { return true; }; @@ -41,8 +41,10 @@ pub fn manifest_has_arch(arch: Arch) -> bool { for (i, b) in TARGET_MATRIX_MANIFEST.bytes().enumerate() { if b == b'"' { if in_quote { - let value = &TARGET_MATRIX_MANIFEST[start..i]; - if matches_target_arch(value, arch) { + if TARGET_MATRIX_MANIFEST + .get(start..i) + .is_some_and(|value| matches_target_arch(value, arch)) + { return true; } } else { diff --git a/src/secret.rs b/src/secret.rs index 2d201f9c..7156e2a3 100644 --- a/src/secret.rs +++ b/src/secret.rs @@ -237,7 +237,7 @@ impl Drop for ZeroizingBytes { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_fixed_stack(input: [u8; 32]) -> u8 { +pub(crate) fn diag_zeroize_fixed_stack(input: [u8; 32]) -> u8 { let secret = SecretBytes::new(input); core::hint::black_box(secret.as_bytes()[0]) } @@ -246,7 +246,7 @@ pub fn diag_zeroize_fixed_stack(input: [u8; 32]) -> u8 { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_fixed_move(input: [u8; 32]) -> u8 { +pub(crate) fn diag_zeroize_fixed_move(input: [u8; 32]) -> u8 { let secret = SecretBytes::new(input); let mut exposed = secret.expose(); let output = core::hint::black_box(exposed[0]); @@ -258,7 +258,7 @@ pub fn diag_zeroize_fixed_move(input: [u8; 32]) -> u8 { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_early_return(input: [u8; 32], stop: bool) -> u8 { +pub(crate) fn diag_zeroize_early_return(input: [u8; 32], stop: bool) -> u8 { let secret = SecretBytes::new(input); if core::hint::black_box(stop) { return 0; @@ -270,7 +270,7 @@ pub fn diag_zeroize_early_return(input: [u8; 32], stop: bool) -> u8 { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_variable_heap(input: Vec) -> usize { +pub(crate) fn diag_zeroize_variable_heap(input: Vec) -> usize { let secret = SecretVec::new(input); core::hint::black_box(secret.len()) } diff --git a/src/traits/aead.rs b/src/traits/aead.rs index b81723fa..e4968e9a 100644 --- a/src/traits/aead.rs +++ b/src/traits/aead.rs @@ -15,6 +15,7 @@ use crate::aead::{AeadBufferError, AeadNonce, OpenError, SealError}; /// Zero-sized capability proving that a call came through a crate-owned nonce /// issuer or the explicit expert extension. #[doc(hidden)] +#[derive(Debug)] pub struct SealToken(()); impl SealToken { diff --git a/src/traits/ct.rs b/src/traits/ct.rs index 71bbf665..cce1cc8c 100644 --- a/src/traits/ct.rs +++ b/src/traits/ct.rs @@ -15,9 +15,37 @@ pub struct CtDecision { mask: u8, } +impl core::fmt::Debug for CtDecision { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("CtDecision(..)") + } +} + impl CtDecision { const TRUE_MASK: u8 = u8::MAX; + #[cfg(any( + test, + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "argon2", + feature = "ascon-aead", + feature = "blake3", + feature = "chacha20poly1305", + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + feature = "hmac", + feature = "hmac-sha3", + feature = "kmac", + feature = "ml-kem", + feature = "poly1305", + feature = "rsa", + feature = "scrypt", + feature = "x25519", + feature = "xchacha20poly1305" + ))] #[inline(always)] const fn from_difference(difference: u64) -> Self { let nonzero = ((difference | difference.wrapping_neg()) >> 63) as u8; @@ -37,14 +65,14 @@ impl CtDecision { self.mask == Self::TRUE_MASK } + #[cfg(any(feature = "ed25519", feature = "kmac", feature = "rsa"))] #[inline(always)] - #[allow(dead_code)] pub(crate) const fn into_u8(self) -> u8 { self.mask & 1 } + #[cfg(feature = "ml-kem")] #[inline(always)] - #[allow(dead_code)] pub(crate) const fn into_mask(self) -> u8 { self.mask } @@ -83,22 +111,40 @@ impl core::ops::Not for CtDecision { } } +#[cfg(any( + test, + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "argon2", + feature = "ascon-aead", + feature = "blake3", + feature = "chacha20poly1305", + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + feature = "hmac", + feature = "hmac-sha3", + feature = "kmac", + feature = "ml-kem", + feature = "poly1305", + feature = "rsa", + feature = "scrypt", + feature = "x25519", + feature = "xchacha20poly1305" +))] #[inline(always)] fn byte_difference(left: &[u8], right: &[u8]) -> u64 { let mut difference = 0u64; - let mut left_chunks = left.chunks_exact(8); - let mut right_chunks = right.chunks_exact(8); - - for (left_chunk, right_chunk) in left_chunks.by_ref().zip(right_chunks.by_ref()) { - let (Ok(left_bytes), Ok(right_bytes)) = (<&[u8; 8]>::try_from(left_chunk), <&[u8; 8]>::try_from(right_chunk)) - else { - return u64::MAX; - }; - difference |= u64::from_ne_bytes(*left_bytes) ^ u64::from_ne_bytes(*right_bytes); + let (left_chunks, left_remainder) = left.as_chunks::<8>(); + let (right_chunks, right_remainder) = right.as_chunks::<8>(); + + for (left_chunk, right_chunk) in left_chunks.iter().zip(right_chunks) { + difference |= u64::from_ne_bytes(*left_chunk) ^ u64::from_ne_bytes(*right_chunk); } let mut remainder = 0u8; - for (left_byte, right_byte) in left_chunks.remainder().iter().zip(right_chunks.remainder()) { + for (left_byte, right_byte) in left_remainder.iter().zip(right_remainder) { remainder |= left_byte ^ right_byte; } difference | u64::from(remainder) @@ -110,8 +156,27 @@ fn byte_difference(left: &[u8], right: &[u8]) -> u64 { /// concrete cryptographic type that owns the bytes. Optimized machine-code /// evidence is tracked separately in `ct.toml`; source structure is not a /// universal constant-time guarantee. +#[cfg(any( + test, + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + all(feature = "argon2", feature = "phc-strings"), + feature = "ascon-aead", + feature = "blake3", + feature = "chacha20poly1305", + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + feature = "hmac", + feature = "hmac-sha3", + feature = "ml-kem", + feature = "poly1305", + all(feature = "scrypt", feature = "phc-strings"), + feature = "x25519", + feature = "xchacha20poly1305" +))] #[inline(always)] -#[allow(dead_code)] pub(crate) fn fixed_eq(left: &[u8; N], right: &[u8; N]) -> CtDecision { // SECURITY: Keep the accumulated word opaque before declassification. LLVM can otherwise fold // equality into target-specific vector reductions; exact binary evidence still owns the @@ -124,8 +189,15 @@ pub(crate) fn fixed_eq(left: &[u8; N], right: &[u8; N]) -> CtDec /// Length mismatch is intentionally observable. Equal-length contents are /// traversed without content-dependent exits. Keep callers individually /// classified in `ct.toml`; fixed-shape owner types must use [`fixed_eq`]. +#[cfg(any( + test, + feature = "argon2", + feature = "kmac", + feature = "ml-kem", + feature = "rsa", + feature = "scrypt" +))] #[inline] -#[allow(dead_code)] pub(crate) fn public_len_eq(left: &[u8], right: &[u8]) -> CtDecision { if left.len() != right.len() { return CtDecision::from_difference(1); @@ -175,6 +247,18 @@ pub fn zeroize(buf: &mut [u8]) { core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } +#[cfg(any( + feature = "aes-gcm", + feature = "ascon-aead", + feature = "blake2b", + feature = "blake2s", + feature = "blake3", + feature = "chacha20poly1305", + feature = "poly1305", + feature = "sha2", + feature = "sha3", + feature = "xchacha20poly1305" +))] mod word_zero_sealed { /// Marker for primitive integer types whose zero representation is /// `0` and whose `write_volatile` of zero is a sound clear. @@ -182,8 +266,7 @@ mod word_zero_sealed { /// Sealed: only the integer types listed here are accepted as scratch /// types for [`zeroize_words_no_fence`] / [`zeroize_words`]. New /// implementors must be reviewed for soundness (no padding, no Drop). - #[allow(dead_code)] - pub trait WordZero: Copy { + pub(crate) trait WordZero: Copy { const ZERO: Self; } @@ -207,6 +290,18 @@ mod word_zero_sealed { } } +#[cfg(any( + feature = "aes-gcm", + feature = "ascon-aead", + feature = "blake2b", + feature = "blake2s", + feature = "blake3", + feature = "chacha20poly1305", + feature = "poly1305", + feature = "sha2", + feature = "sha3", + feature = "xchacha20poly1305" +))] pub(crate) use word_zero_sealed::WordZero; /// Volatile-zero a slice of `WordZero` integers without a compiler fence. @@ -216,8 +311,19 @@ pub(crate) use word_zero_sealed::WordZero; /// loops over `core::ptr::write_volatile` patterns. Caller is responsible /// for emitting a single `compiler_fence(SeqCst)` after all related /// zeroizations. +#[cfg(any( + feature = "aes-gcm", + feature = "ascon-aead", + feature = "blake2b", + feature = "blake2s", + feature = "blake3", + feature = "chacha20poly1305", + feature = "poly1305", + feature = "sha2", + feature = "sha3", + feature = "xchacha20poly1305" +))] #[inline(always)] -#[allow(dead_code)] pub(crate) fn zeroize_words_no_fence(words: &mut [T]) { for word in words { // SAFETY: `word` is a valid, aligned, dereferenceable pointer to `T`. @@ -228,8 +334,14 @@ pub(crate) fn zeroize_words_no_fence(words: &mut [T]) { } /// Volatile-zero a slice of `WordZero` integers and emit a compiler fence. +#[cfg(any( + feature = "aes-gcm", + feature = "ascon-aead", + feature = "blake3", + feature = "sha2", + feature = "sha3" +))] #[inline(always)] -#[allow(dead_code)] pub(crate) fn zeroize_words(words: &mut [T]) { zeroize_words_no_fence(words); core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); diff --git a/src/traits/error.rs b/src/traits/error.rs index 96045ac3..f6fb9b71 100644 --- a/src/traits/error.rs +++ b/src/traits/error.rs @@ -78,9 +78,12 @@ mod tests { #[test] fn is_clone() { + fn clone_via_trait_bound(value: &T) -> T { + value.clone() + } + let e = VerificationError::new(); - #[allow(clippy::clone_on_copy)] - let cloned = e.clone(); + let cloned = clone_via_trait_bound(&e); assert_eq!(e, cloned); } @@ -110,7 +113,7 @@ mod tests { fn verify_match() -> Result<(), VerificationError> { Ok(()) } - assert!(verify_match().is_ok()); + assert_eq!(verify_match(), Ok(())); } #[test] @@ -127,7 +130,7 @@ mod tests { fn returns_err() -> Result<(), VerificationError> { Err(VerificationError::new()) } - let err = returns_err().unwrap_err(); + let err = returns_err().expect_err("returns_err must return VerificationError"); assert_eq!(err.to_string(), "verification failed"); } diff --git a/src/traits/io.rs b/src/traits/io.rs index 70f50960..ddc3d670 100644 --- a/src/traits/io.rs +++ b/src/traits/io.rs @@ -179,6 +179,7 @@ impl core::fmt::Debu #[cfg(feature = "std")] impl ChecksumReader { + /// Wrap `inner` with a checksum initialized to its algorithm-defined default. #[inline] #[must_use] pub fn new(inner: R) -> Self { @@ -188,6 +189,7 @@ impl ChecksumReader { } } + /// Wrap `inner` with a checksum initialized from `initial`. #[inline] #[must_use] pub fn with_initial(inner: R, initial: C::Output) -> Self { @@ -197,32 +199,42 @@ impl ChecksumReader { } } + /// Return the checksum of the bytes read through this adapter so far. #[inline] #[must_use] pub fn checksum(&self) -> C::Output { self.hasher.finalize() } + /// Borrow the checksum state mutably. + /// + /// Direct updates affect the returned checksum but do not read from the inner reader. #[inline] pub fn hasher_mut(&mut self) -> &mut C { &mut self.hasher } + /// Consume the adapter and return the inner reader and current checksum. #[inline] pub fn into_parts(self) -> (R, C::Output) { (self.inner, self.hasher.finalize()) } + /// Consume the adapter and return the inner reader, discarding the checksum state. #[inline] pub fn into_inner(self) -> R { self.inner } + /// Borrow the inner reader. #[inline] pub fn inner(&self) -> &R { &self.inner } + /// Borrow the inner reader mutably. + /// + /// Bytes read directly from this value are not included in the checksum. #[inline] pub fn inner_mut(&mut self) -> &mut R { &mut self.inner @@ -303,6 +315,7 @@ impl core::fmt::Debu #[cfg(feature = "std")] impl ChecksumWriter { + /// Wrap `inner` with a checksum initialized to its algorithm-defined default. #[inline] #[must_use] pub fn new(inner: W) -> Self { @@ -312,6 +325,7 @@ impl ChecksumWriter { } } + /// Wrap `inner` with a checksum initialized from `initial`. #[inline] #[must_use] pub fn with_initial(inner: W, initial: C::Output) -> Self { @@ -321,32 +335,42 @@ impl ChecksumWriter { } } + /// Return the checksum of the bytes written through this adapter so far. #[inline] #[must_use] pub fn checksum(&self) -> C::Output { self.hasher.finalize() } + /// Borrow the checksum state mutably. + /// + /// Direct updates affect the returned checksum but do not write to the inner writer. #[inline] pub fn hasher_mut(&mut self) -> &mut C { &mut self.hasher } + /// Consume the adapter and return the inner writer and current checksum. #[inline] pub fn into_parts(self) -> (W, C::Output) { (self.inner, self.hasher.finalize()) } + /// Consume the adapter and return the inner writer, discarding the checksum state. #[inline] pub fn into_inner(self) -> W { self.inner } + /// Borrow the inner writer. #[inline] pub fn inner(&self) -> &W { &self.inner } + /// Borrow the inner writer mutably. + /// + /// Bytes written directly to this value are not included in the checksum. #[inline] pub fn inner_mut(&mut self) -> &mut W { &mut self.inner @@ -424,6 +448,7 @@ impl core::fmt #[cfg(feature = "std")] impl DigestReader { + /// Wrap `inner` with a new digest state. #[inline] #[must_use] pub fn new(inner: R) -> Self { @@ -433,32 +458,42 @@ impl DigestReader { } } + /// Return the digest of the bytes read through this adapter so far. #[inline] #[must_use] pub fn digest(&self) -> D::Output { self.hasher.finalize() } + /// Borrow the digest state mutably. + /// + /// Direct updates affect the returned digest but do not read from the inner reader. #[inline] pub fn hasher_mut(&mut self) -> &mut D { &mut self.hasher } + /// Consume the adapter and return the inner reader and current digest. #[inline] pub fn into_parts(self) -> (R, D::Output) { (self.inner, self.hasher.finalize()) } + /// Consume the adapter and return the inner reader, discarding the digest state. #[inline] pub fn into_inner(self) -> R { self.inner } + /// Borrow the inner reader. #[inline] pub fn inner(&self) -> &R { &self.inner } + /// Borrow the inner reader mutably. + /// + /// Bytes read directly from this value are not included in the digest. #[inline] pub fn inner_mut(&mut self) -> &mut R { &mut self.inner @@ -538,6 +573,7 @@ impl core::fmt #[cfg(feature = "std")] impl DigestWriter { + /// Wrap `inner` with a new digest state. #[inline] #[must_use] pub fn new(inner: W) -> Self { @@ -547,32 +583,42 @@ impl DigestWriter { } } + /// Return the digest of the bytes written through this adapter so far. #[inline] #[must_use] pub fn digest(&self) -> D::Output { self.hasher.finalize() } + /// Borrow the digest state mutably. + /// + /// Direct updates affect the returned digest but do not write to the inner writer. #[inline] pub fn hasher_mut(&mut self) -> &mut D { &mut self.hasher } + /// Consume the adapter and return the inner writer and current digest. #[inline] pub fn into_parts(self) -> (W, D::Output) { (self.inner, self.hasher.finalize()) } + /// Consume the adapter and return the inner writer, discarding the digest state. #[inline] pub fn into_inner(self) -> W { self.inner } + /// Borrow the inner writer. #[inline] pub fn inner(&self) -> &W { &self.inner } + /// Borrow the inner writer mutably. + /// + /// Bytes written directly to this value are not included in the digest. #[inline] pub fn inner_mut(&mut self) -> &mut W { &mut self.inner @@ -717,7 +763,9 @@ mod tests { #[test] fn checksum_writer_hashes_only_written_prefix() { let mut writer = ChecksumWriter::<_, Sum>::new(PartialWriter::new(4)); - let written = writer.write(b"abcdef").unwrap(); + let written = writer + .write(b"abcdef") + .expect("in-memory checksum writer must accept input"); assert_eq!(written, 4); assert_eq!(writer.checksum(), checksum_sum(b"abcd")); @@ -730,7 +778,9 @@ mod tests { fn checksum_writer_vectored_hashes_only_written_prefix() { let mut writer = ChecksumWriter::<_, Sum>::new(PartialWriter::new(5)); let bufs = [IoSlice::new(b"ab"), IoSlice::new(b"cdef"), IoSlice::new(b"gh")]; - let written = writer.write_vectored(&bufs).unwrap(); + let written = writer + .write_vectored(&bufs) + .expect("in-memory checksum writer must accept vectored input"); assert_eq!(written, 5); assert_eq!(writer.checksum(), checksum_sum(b"abcde")); @@ -742,7 +792,9 @@ mod tests { #[test] fn digest_writer_hashes_only_written_prefix() { let mut writer = DigestWriter::<_, SumDigest>::new(PartialWriter::new(3)); - let written = writer.write(b"abcdef").unwrap(); + let written = writer + .write(b"abcdef") + .expect("in-memory digest writer must accept input"); assert_eq!(written, 3); assert_eq!(writer.digest(), digest_sum(b"abc")); @@ -755,7 +807,9 @@ mod tests { fn digest_writer_vectored_hashes_only_written_prefix() { let mut writer = DigestWriter::<_, SumDigest>::new(PartialWriter::new(6)); let bufs = [IoSlice::new(b"ab"), IoSlice::new(b"cdef"), IoSlice::new(b"ghij")]; - let written = writer.write_vectored(&bufs).unwrap(); + let written = writer + .write_vectored(&bufs) + .expect("in-memory digest writer must accept vectored input"); assert_eq!(written, 6); assert_eq!(writer.digest(), digest_sum(b"abcdef")); @@ -776,7 +830,9 @@ mod tests { IoSliceMut::new(&mut third), ]; - let read = reader.read_vectored(&mut bufs).unwrap(); + let read = reader + .read_vectored(&mut bufs) + .expect("in-memory checksum reader must provide vectored input"); assert_eq!(read, 5); assert_eq!(second, *b"ab"); assert_eq!(third, *b"cde"); @@ -795,7 +851,9 @@ mod tests { IoSliceMut::new(&mut third), ]; - let read = reader.read_vectored(&mut bufs).unwrap(); + let read = reader + .read_vectored(&mut bufs) + .expect("in-memory digest reader must provide vectored input"); assert_eq!(read, 5); assert_eq!(second, *b"ab"); assert_eq!(third, *b"cde"); @@ -806,7 +864,9 @@ mod tests { fn checksum_writer_vectored_skips_empty_buffers() { let mut writer = ChecksumWriter::<_, Sum>::new(PartialWriter::new(5)); let bufs = [IoSlice::new(b""), IoSlice::new(b"ab"), IoSlice::new(b"cde")]; - let written = writer.write_vectored(&bufs).unwrap(); + let written = writer + .write_vectored(&bufs) + .expect("in-memory checksum writer must accept nonempty vectored input"); assert_eq!(written, 5); assert_eq!(writer.checksum(), checksum_sum(b"abcde")); @@ -820,7 +880,9 @@ mod tests { fn digest_writer_vectored_skips_empty_buffers() { let mut writer = DigestWriter::<_, SumDigest>::new(PartialWriter::new(5)); let bufs = [IoSlice::new(b""), IoSlice::new(b"ab"), IoSlice::new(b"cde")]; - let written = writer.write_vectored(&bufs).unwrap(); + let written = writer + .write_vectored(&bufs) + .expect("in-memory digest writer must accept nonempty vectored input"); assert_eq!(written, 5); assert_eq!(writer.digest(), digest_sum(b"abcde")); diff --git a/tests/aead_foundations.rs b/tests/aead_foundations.rs index 7e583af7..e5b016ae 100644 --- a/tests/aead_foundations.rs +++ b/tests/aead_foundations.rs @@ -71,10 +71,14 @@ fn aead_encrypt_and_decrypt_helpers_round_trip() { let aead = fixture_cipher(); let mut sealed = [0u8; 12 + ChaCha20Poly1305::TAG_SIZE]; - aead.encrypt(&nonce, aad, &plaintext, &mut sealed).unwrap(); + aead + .encrypt(&nonce, aad, &plaintext, &mut sealed) + .expect("combined AEAD seal buffer must fit plaintext and tag"); let mut opened = [0u8; 12]; - aead.decrypt(&nonce, aad, &sealed, &mut opened).unwrap(); + aead + .decrypt(&nonce, aad, &sealed, &mut opened) + .expect("freshly sealed combined AEAD ciphertext must authenticate"); assert_eq!(opened, plaintext); } @@ -86,15 +90,21 @@ fn aead_seal_random_issues_fresh_nonces_and_round_trips() { let aead = fixture_cipher(); let mut sealed_a = [0u8; 12 + ChaCha20Poly1305::TAG_SIZE]; - let nonce_a = aead.seal_random(aad, b"hello world!", &mut sealed_a).unwrap(); + let nonce_a = aead + .seal_random(aad, b"hello world!", &mut sealed_a) + .expect("first random-nonce AEAD seal must succeed"); let mut sealed_b = [0u8; 12 + ChaCha20Poly1305::TAG_SIZE]; - let nonce_b = aead.seal_random(aad, b"hello world!", &mut sealed_b).unwrap(); + let nonce_b = aead + .seal_random(aad, b"hello world!", &mut sealed_b) + .expect("second random-nonce AEAD seal must succeed"); assert_ne!(nonce_a, nonce_b, "successive random seals must not reuse a nonce"); let mut opened = [0u8; 12]; - aead.decrypt(&nonce_a, aad, &sealed_a, &mut opened).unwrap(); + aead + .decrypt(&nonce_a, aad, &sealed_a, &mut opened) + .expect("fresh random-nonce AEAD ciphertext must authenticate"); assert_eq!(&opened, b"hello world!"); } @@ -105,10 +115,14 @@ fn aead_seal_random_in_place_returns_nonce_and_tag_for_open() { let aead = fixture_cipher(); let mut buffer = *b"detached"; - let (nonce, tag) = aead.seal_random_in_place(aad, &mut buffer).unwrap(); + let (nonce, tag) = aead + .seal_random_in_place(aad, &mut buffer) + .expect("random-nonce detached AEAD seal must succeed"); assert_ne!(&buffer, b"detached"); - aead.decrypt_in_place(&nonce, aad, &mut buffer, &tag).unwrap(); + aead + .decrypt_in_place(&nonce, aad, &mut buffer, &tag) + .expect("fresh random-nonce detached ciphertext must authenticate"); assert_eq!(&buffer, b"detached"); } @@ -118,7 +132,9 @@ fn aead_seal_random_maps_buffer_errors_without_mutating_output() { let aead = fixture_cipher(); let mut out = [0xA5; 3]; - let err = aead.seal_random(b"", b"data", &mut out).unwrap_err(); + let err = aead + .seal_random(b"", b"data", &mut out) + .expect_err("undersized random-nonce AEAD output must be rejected"); assert_eq!(err, RandomSealError::seal(SealError::buffer())); assert_eq!( @@ -137,15 +153,19 @@ fn detached_aliases_match_core_behavior() { let mut left = *b"detached"; let mut right = left; - let tag_left = aead.encrypt_in_place(&nonce, aad, &mut left).unwrap(); - let tag_right = aead.encrypt_in_place_detached(&nonce, aad, &mut right).unwrap(); + let tag_left = aead + .encrypt_in_place(&nonce, aad, &mut left) + .expect("core detached AEAD seal must succeed"); + let tag_right = aead + .encrypt_in_place_detached(&nonce, aad, &mut right) + .expect("detached AEAD alias must seal the same input"); assert_eq!(left, right); assert!(tag_left.ct_eq(&tag_right).declassify()); aead .decrypt_in_place_detached(&nonce, aad, &mut right, &tag_right) - .unwrap(); + .expect("detached AEAD alias must open its fresh ciphertext"); assert_eq!(right, *b"detached"); } @@ -156,7 +176,9 @@ fn aead_open_reports_buffer_and_verification_failures() { let aead = fixture_cipher(); let mut sealed = [0u8; 4 + ChaCha20Poly1305::TAG_SIZE]; - aead.encrypt(&nonce, b"aad", b"data", &mut sealed).unwrap(); + aead + .encrypt(&nonce, b"aad", b"data", &mut sealed) + .expect("AEAD error fixture seal buffer must fit plaintext and tag"); let mut short_out = [0u8; 3]; assert_eq!( @@ -184,8 +206,12 @@ fn aead_combined_open_accepts_empty_plaintext() { let mut sealed = [0u8; ChaCha20Poly1305::TAG_SIZE]; let mut opened = [0u8; 0]; - aead.encrypt(&nonce, b"aad", b"", &mut sealed).unwrap(); - aead.decrypt(&nonce, b"aad", &sealed, &mut opened).unwrap(); + aead + .encrypt(&nonce, b"aad", b"", &mut sealed) + .expect("AEAD must seal an empty plaintext"); + aead + .decrypt(&nonce, b"aad", &sealed, &mut opened) + .expect("AEAD must open its empty-plaintext ciphertext"); } #[test] @@ -210,7 +236,9 @@ fn all_aeads_clear_in_place_buffer_on_verification_failure() { let cipher = $cipher; let nonce = $nonce; let mut buffer = *b"failed-open-clear"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect(concat!($name, " failed-open fixture must seal")); let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0x80; let bad_tag = <$tag>::from_bytes(bad_tag); @@ -282,7 +310,13 @@ fn all_aeads_clear_in_place_buffer_on_verification_failure() { #[test] #[cfg(feature = "aead")] fn aead_length_helpers_reject_invalid_sizes() { - assert_eq!(ChaCha20Poly1305::ciphertext_len(5).unwrap(), 21); - assert_eq!(ChaCha20Poly1305::plaintext_len(21).unwrap(), 5); + assert_eq!( + ChaCha20Poly1305::ciphertext_len(5).expect("five-byte plaintext length must fit ChaCha20-Poly1305"), + 21 + ); + assert_eq!( + ChaCha20Poly1305::plaintext_len(21).expect("21-byte ciphertext length must include a full tag"), + 5 + ); assert_eq!(ChaCha20Poly1305::plaintext_len(15), Err(AeadBufferError::new())); } diff --git a/tests/aead_kernel_equivalence.rs b/tests/aead_kernel_equivalence.rs index 3693fabc..f5ec44dc 100644 --- a/tests/aead_kernel_equivalence.rs +++ b/tests/aead_kernel_equivalence.rs @@ -21,7 +21,7 @@ use rscrypto::{ }; /// Function pointer type for ChaCha20 XOR-keystream kernels. -type XorKeystreamFn = fn(&[u8; 32], u32, &[u8; 12], &mut [u8]); +type XorKeystreamFn = unsafe fn(&[u8; 32], u32, &[u8; 12], &mut [u8]); #[derive(Clone, Copy)] struct Backend { @@ -119,7 +119,8 @@ fn all_chacha20_backends_match_portable_at_counter_zero() { for backend in runnable_backends() { let mut actual = plain.clone(); - (backend.xor_keystream)(&key, 0, &nonce, &mut actual); + // SAFETY: `runnable_backends` proves the exact capability set; `TEST_SIZES` uses at most 65 blocks from zero. + unsafe { (backend.xor_keystream)(&key, 0, &nonce, &mut actual) }; assert_eq!( actual, expected, "ChaCha20 backend {} diverged from portable at len={len}, counter=0", @@ -152,7 +153,9 @@ fn all_chacha20_backends_match_portable_at_arbitrary_counters() { for backend in runnable_backends() { let mut actual = plain.clone(); - (backend.xor_keystream)(&key, counter, &nonce, &mut actual); + // SAFETY: `runnable_backends` proves the exact capability set; at most 16 blocks from the largest tested + // counter, 0x1000_0000, stays within the `u32` counter range. + unsafe { (backend.xor_keystream)(&key, counter, &nonce, &mut actual) }; assert_eq!( actual, expected, "ChaCha20 backend {} diverged from portable at len={len}, counter={counter}", @@ -176,8 +179,11 @@ fn all_chacha20_backends_self_inverse() { for backend in runnable_backends() { let mut buffer = original.clone(); - (backend.xor_keystream)(&key, 0, &nonce, &mut buffer); - (backend.xor_keystream)(&key, 0, &nonce, &mut buffer); + // SAFETY: `runnable_backends` proves the exact capability set; `TEST_SIZES` uses at most 65 blocks from zero. + unsafe { + (backend.xor_keystream)(&key, 0, &nonce, &mut buffer); + (backend.xor_keystream)(&key, 0, &nonce, &mut buffer); + } assert_eq!( buffer, original, "ChaCha20 backend {} not self-inverse at len={len}", diff --git a/tests/aead_wycheproof.rs b/tests/aead_wycheproof.rs index 16db55de..0be63f26 100644 --- a/tests/aead_wycheproof.rs +++ b/tests/aead_wycheproof.rs @@ -37,9 +37,7 @@ struct AeadCase { } fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + value[name].as_str().expect(name) } fn groups(suite: &Value) -> &[Value] { @@ -53,6 +51,11 @@ fn tests(group: &Value) -> &[Value] { } fn case_from_json(test: &Value) -> AeadCase { + let result = field(test, "result"); + assert!( + matches!(result, "valid" | "invalid"), + "unsupported Wycheproof AEAD result `{result}`" + ); AeadCase { tc_id: test["tcId"].as_u64().expect("tcId must be numeric"), comment: field(test, "comment").to_owned(), @@ -62,11 +65,7 @@ fn case_from_json(test: &Value) -> AeadCase { msg: decode_hex_vec(field(test, "msg")), ct: decode_hex_vec(field(test, "ct")), tag: decode_hex_vec(field(test, "tag")), - valid: match field(test, "result") { - "valid" => true, - "invalid" => false, - other => panic!("unsupported Wycheproof AEAD result `{other}`"), - }, + valid: result == "valid", } } @@ -80,14 +79,34 @@ fn assert_open_failure_clears_buffer(case: &AeadCase, buffer: &[u8]) { } fn assert_chacha20poly1305_case(case: &AeadCase) { - let key = ChaCha20Poly1305Key::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce96::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = ChaCha20Poly1305Tag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = ChaCha20Poly1305Key::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof ChaCha20-Poly1305 key must be 32 bytes"), + ); + let nonce = Nonce96::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof ChaCha20-Poly1305 nonce must be 12 bytes"), + ); + let tag = ChaCha20Poly1305Tag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof ChaCha20-Poly1305 tag must be 16 bytes"), + ); let cipher = ChaCha20Poly1305::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof ChaCha20-Poly1305 vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof ChaCha20-Poly1305 tcId {} opened incorrectly", @@ -95,7 +114,9 @@ fn assert_chacha20poly1305_case(case: &AeadCase) { ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof ChaCha20-Poly1305 vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof ChaCha20-Poly1305 tcId {} encrypted incorrectly", @@ -114,14 +135,34 @@ fn assert_chacha20poly1305_case(case: &AeadCase) { } fn assert_xchacha20poly1305_case(case: &AeadCase) { - let key = XChaCha20Poly1305Key::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce192::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = XChaCha20Poly1305Tag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = XChaCha20Poly1305Key::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof XChaCha20-Poly1305 key must be 32 bytes"), + ); + let nonce = Nonce192::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof XChaCha20-Poly1305 nonce must be 24 bytes"), + ); + let tag = XChaCha20Poly1305Tag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof XChaCha20-Poly1305 tag must be 16 bytes"), + ); let cipher = XChaCha20Poly1305::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof XChaCha20-Poly1305 vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof XChaCha20-Poly1305 tcId {} opened incorrectly", @@ -129,7 +170,9 @@ fn assert_xchacha20poly1305_case(case: &AeadCase) { ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof XChaCha20-Poly1305 vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof XChaCha20-Poly1305 tcId {} encrypted incorrectly", @@ -148,21 +191,43 @@ fn assert_xchacha20poly1305_case(case: &AeadCase) { } fn assert_aes128gcm_case(case: &AeadCase) { - let key = Aes128GcmKey::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce96::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = Aes128GcmTag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = Aes128GcmKey::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof AES-128-GCM key must be 16 bytes"), + ); + let nonce = Nonce96::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof AES-GCM nonce must be 12 bytes"), + ); + let tag = Aes128GcmTag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof AES-128-GCM tag must be 16 bytes"), + ); let cipher = Aes128Gcm::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof AES-128-GCM vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof AES-128-GCM tcId {} opened incorrectly", case.tc_id ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof AES-128-GCM vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof AES-128-GCM tcId {} encrypted incorrectly", @@ -181,21 +246,43 @@ fn assert_aes128gcm_case(case: &AeadCase) { } fn assert_aes256gcm_case(case: &AeadCase) { - let key = Aes256GcmKey::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce96::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = Aes256GcmTag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = Aes256GcmKey::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof AES-256-GCM key must be 32 bytes"), + ); + let nonce = Nonce96::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof AES-GCM nonce must be 12 bytes"), + ); + let tag = Aes256GcmTag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof AES-256-GCM tag must be 16 bytes"), + ); let cipher = Aes256Gcm::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof AES-256-GCM vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof AES-256-GCM tcId {} opened incorrectly", case.tc_id ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof AES-256-GCM vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof AES-256-GCM tcId {} encrypted incorrectly", @@ -214,21 +301,43 @@ fn assert_aes256gcm_case(case: &AeadCase) { } fn assert_aes128gcmsiv_case(case: &AeadCase) { - let key = Aes128GcmSivKey::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce96::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = Aes128GcmSivTag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = Aes128GcmSivKey::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof AES-128-GCM-SIV key must be 16 bytes"), + ); + let nonce = Nonce96::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof AES-GCM-SIV nonce must be 12 bytes"), + ); + let tag = Aes128GcmSivTag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof AES-128-GCM-SIV tag must be 16 bytes"), + ); let cipher = Aes128GcmSiv::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof AES-128-GCM-SIV vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof AES-128-GCM-SIV tcId {} opened incorrectly", case.tc_id ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof AES-128-GCM-SIV vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof AES-128-GCM-SIV tcId {} encrypted incorrectly", @@ -247,21 +356,43 @@ fn assert_aes128gcmsiv_case(case: &AeadCase) { } fn assert_aes256gcmsiv_case(case: &AeadCase) { - let key = Aes256GcmSivKey::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce96::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = Aes256GcmSivTag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = Aes256GcmSivKey::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof AES-256-GCM-SIV key must be 32 bytes"), + ); + let nonce = Nonce96::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof AES-GCM-SIV nonce must be 12 bytes"), + ); + let tag = Aes256GcmSivTag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof AES-256-GCM-SIV tag must be 16 bytes"), + ); let cipher = Aes256GcmSiv::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof AES-256-GCM-SIV vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof AES-256-GCM-SIV tcId {} opened incorrectly", case.tc_id ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof AES-256-GCM-SIV vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof AES-256-GCM-SIV tcId {} encrypted incorrectly", @@ -280,21 +411,43 @@ fn assert_aes256gcmsiv_case(case: &AeadCase) { } fn assert_aegis256_case(case: &AeadCase) { - let key = Aegis256Key::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce256::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = Aegis256Tag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = Aegis256Key::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof AEGIS-256 key must be 32 bytes"), + ); + let nonce = Nonce256::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof AEGIS-256 nonce must be 32 bytes"), + ); + let tag = Aegis256Tag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof AEGIS-256 tag must be 16 bytes"), + ); let cipher = Aegis256::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof AEGIS-256 vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof AEGIS-256 tcId {} opened incorrectly", case.tc_id ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof AEGIS-256 vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof AEGIS-256 tcId {} encrypted incorrectly", @@ -352,10 +505,17 @@ fn wycheproof_aes_gcm_open_vectors_match_expected_results() { && matches!(group["keySize"].as_u64(), Some(128 | 256)) && group["tagSize"].as_u64() == Some(128) }, - |case| match case.key.len() { - 16 => assert_aes128gcm_case(case), - 32 => assert_aes256gcm_case(case), - len => panic!("unsupported AES-GCM key length {len}"), + |case| { + assert!( + matches!(case.key.len(), 16 | 32), + "unsupported AES-GCM key length {}", + case.key.len() + ); + if case.key.len() == 16 { + assert_aes128gcm_case(case); + } else { + assert_aes256gcm_case(case); + } }, ); } @@ -374,10 +534,17 @@ fn wycheproof_aes_gcm_siv_open_vectors_match_expected_results() { && matches!(group["keySize"].as_u64(), Some(128 | 256)) && group["tagSize"].as_u64() == Some(128) }, - |case| match case.key.len() { - 16 => assert_aes128gcmsiv_case(case), - 32 => assert_aes256gcmsiv_case(case), - len => panic!("unsupported AES-GCM-SIV key length {len}"), + |case| { + assert!( + matches!(case.key.len(), 16 | 32), + "unsupported AES-GCM-SIV key length {}", + case.key.len() + ); + if case.key.len() == 16 { + assert_aes128gcmsiv_case(case); + } else { + assert_aes256gcmsiv_case(case); + } }, ); } diff --git a/tests/aegis256_oracle.rs b/tests/aegis256_oracle.rs index afbd757a..3056239b 100644 --- a/tests/aegis256_oracle.rs +++ b/tests/aegis256_oracle.rs @@ -26,7 +26,9 @@ fn assert_matches_oracle(key_bytes: &[u8; 32], nonce_bytes: &[u8; 32], aad: &[u8 // Encrypt with rscrypto. let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto must seal valid AEGIS-256 oracle input"); // Encrypt with oracle (consumes self, returns (Vec, [u8; 16])). let (oracle_ct, oracle_tag) = Oracle::<16>::new(key_bytes, nonce_bytes).encrypt(plaintext, aad); @@ -35,13 +37,15 @@ fn assert_matches_oracle(key_bytes: &[u8; 32], nonce_bytes: &[u8; 32], aad: &[u8 assert_eq!(tag.as_bytes(), &oracle_tag, "tag mismatch (len={})", plaintext.len()); // Decrypt with rscrypto. - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AEGIS-256 ciphertext must authenticate"); assert_eq!(ours, plaintext, "decrypt round-trip failed (len={})", plaintext.len()); // Cross-decrypt: oracle decrypts rscrypto's ciphertext. let oracle_pt = Oracle::<16>::new(key_bytes, nonce_bytes) .decrypt(&oracle_ct, &oracle_tag, aad) - .unwrap(); + .expect("AEGIS-256 oracle must authenticate rscrypto ciphertext"); assert_eq!( oracle_pt, plaintext, @@ -94,7 +98,7 @@ fn aegis256_oracle_varied_sizes() { fn aegis256_oracle_large_input() { let key = [0x77u8; 32]; let nonce = [0x88u8; 32]; - let plaintext: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let plaintext: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, b"large", &plaintext); } @@ -107,14 +111,15 @@ fn aegis256_rejects_modified_tag() { let cipher = Aegis256::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("AEGIS-256 tag-forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aegis256Tag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aegis256Tag::from_bytes(tag)) + .expect_err("AEGIS-256 must reject a modified tag"); } #[test] @@ -124,10 +129,14 @@ fn aegis256_rejects_modified_ciphertext() { let cipher = Aegis256::new(&key); let mut buffer = *b"tamper-detect"; - let tag = cipher.encrypt_in_place(&nonce, b"", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"", &mut buffer) + .expect("AEGIS-256 ciphertext-tampering fixture must seal"); buffer[0] ^= 1; - assert!(cipher.decrypt_in_place(&nonce, b"", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"", &mut buffer, &tag) + .expect_err("AEGIS-256 must reject modified ciphertext"); } #[test] @@ -137,7 +146,11 @@ fn aegis256_rejects_wrong_aad() { let cipher = Aegis256::new(&key); let mut buffer = *b"aad-mismatch"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buffer) + .expect("AEGIS-256 AAD-mismatch fixture must seal"); - assert!(cipher.decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag) + .expect_err("AEGIS-256 must reject incorrect associated data"); } diff --git a/tests/aes128gcm_oracle.rs b/tests/aes128gcm_oracle.rs index e8b5378d..07050174 100644 --- a/tests/aes128gcm_oracle.rs +++ b/tests/aes128gcm_oracle.rs @@ -24,10 +24,10 @@ use rscrypto::{ fn deterministic_bytes(seed: u8, len: usize) -> Vec { let mut out = Vec::with_capacity(len); - let mut x = seed as u32; + let mut x = u32::from(seed); for _ in 0..len { x = x.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - out.push((x >> 24) as u8); + out.push(x.to_be_bytes()[0]); } out } @@ -42,13 +42,15 @@ fn assert_matches_oracle(key_bytes: &[u8; 16], nonce_bytes: &[u8; 12], aad: &[u8 // Encrypt with rscrypto. let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto must seal valid AES-128-GCM oracle input"); // Encrypt with oracle. let mut oracle_buf = plaintext.to_vec(); let oracle_tag = oracle .encrypt_inout_detached(&oracle_nonce, aad, oracle_buf.as_mut_slice().into()) - .unwrap(); + .expect("RustCrypto must seal valid AES-128-GCM oracle input"); assert_eq!(ours, oracle_buf, "ciphertext mismatch (len={})", plaintext.len()); assert_eq!( @@ -59,7 +61,9 @@ fn assert_matches_oracle(key_bytes: &[u8; 16], nonce_bytes: &[u8; 12], aad: &[u8 ); // Decrypt with rscrypto. - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AES-128-GCM ciphertext must authenticate"); assert_eq!(ours, plaintext, "decrypt round-trip failed (len={})", plaintext.len()); } @@ -108,7 +112,7 @@ fn aes128gcm_oracle_large_input() { let key = [0x77u8; 16]; let nonce = [0x88u8; 12]; // 8 KiB — exercises multi-block GHASH and CTR paths. - let plaintext: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let plaintext: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, b"large", &plaintext); } @@ -120,8 +124,8 @@ fn aes128gcm_oracle_aad_size_sweep() { // Sweeps cover the wide-GHASH 4-block boundary (64-byte chunks) and the // partial-tail seam at +/-1 around 16-byte block boundaries. - for aad_len in [0, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { - let aad: Vec = (0..aad_len).map(|i| (i & 0xFF) as u8).collect(); + for aad_len in [0usize, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { + let aad: Vec = (0..aad_len).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, &aad, plaintext); } } @@ -132,13 +136,15 @@ fn aes128gcm_oracle_all_short_lengths() { for size in 0..=255usize { let mut nonce = [0x18u8; 12]; - nonce[8..12].copy_from_slice(&(size as u32).to_be_bytes()); + let size_word = u32::try_from(size).expect("short-length case must fit in a 32-bit nonce field"); + let size_byte = size.to_le_bytes()[0]; + nonce[8..12].copy_from_slice(&size_word.to_be_bytes()); let aad = if size % 3 == 0 { Vec::new() } else { - deterministic_bytes(0x42 ^ size as u8, size % 97) + deterministic_bytes(0x42 ^ size_byte, size % 97) }; - let plaintext = deterministic_bytes(0x81 ^ size as u8, size); + let plaintext = deterministic_bytes(0x81 ^ size_byte, size); assert_matches_oracle(&key, &nonce, &aad, &plaintext); } } @@ -150,9 +156,11 @@ fn aes128gcm_oracle_large_aligned_and_unaligned_lengths() { for (case_idx, &size) in [4096usize, 4097, 8191, 8192, 8193, 16_384, 16_385].iter().enumerate() { let mut nonce = [0x71u8; 12]; - nonce[4..8].copy_from_slice(&(case_idx as u32).to_be_bytes()); - nonce[8..12].copy_from_slice(&(size as u32).to_be_bytes()); - let plaintext = deterministic_bytes(0x33 ^ case_idx as u8, size); + let case_word = u32::try_from(case_idx).expect("GCM oracle case index must fit in a 32-bit nonce field"); + let size_word = u32::try_from(size).expect("GCM oracle size must fit in a 32-bit nonce field"); + nonce[4..8].copy_from_slice(&case_word.to_be_bytes()); + nonce[8..12].copy_from_slice(&size_word.to_be_bytes()); + let plaintext = deterministic_bytes(0x33 ^ case_idx.to_le_bytes()[0], size); for aad in &aad_cases { assert_matches_oracle(&key, &nonce, aad, &plaintext); } @@ -168,14 +176,15 @@ fn aes128gcm_rejects_modified_tag() { let cipher = Aes128Gcm::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("AES-128-GCM tag-forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes128GcmTag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes128GcmTag::from_bytes(tag)) + .expect_err("AES-128-GCM must reject a modified tag"); } #[test] @@ -185,10 +194,14 @@ fn aes128gcm_rejects_modified_ciphertext() { let cipher = Aes128Gcm::new(&key); let mut buffer = *b"tamper-detect"; - let tag = cipher.encrypt_in_place(&nonce, b"", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"", &mut buffer) + .expect("AES-128-GCM ciphertext-tampering fixture must seal"); buffer[0] ^= 1; - assert!(cipher.decrypt_in_place(&nonce, b"", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"", &mut buffer, &tag) + .expect_err("AES-128-GCM must reject modified ciphertext"); } #[test] @@ -198,7 +211,11 @@ fn aes128gcm_rejects_wrong_aad() { let cipher = Aes128Gcm::new(&key); let mut buffer = *b"aad-mismatch"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buffer) + .expect("AES-128-GCM AAD-mismatch fixture must seal"); - assert!(cipher.decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag) + .expect_err("AES-128-GCM must reject incorrect associated data"); } diff --git a/tests/aes128gcmsiv_oracle.rs b/tests/aes128gcmsiv_oracle.rs index c84db9f5..19d7443e 100644 --- a/tests/aes128gcmsiv_oracle.rs +++ b/tests/aes128gcmsiv_oracle.rs @@ -14,10 +14,7 @@ #![cfg(feature = "aead")] -use aes_gcm_siv::{ - Aes128GcmSiv as Oracle, KeyInit, - aead::{AeadInPlace, generic_array::GenericArray}, -}; +use aes_gcm_siv::{Aes128GcmSiv as Oracle, KeyInit, Nonce as OracleNonce, aead::AeadInOut}; use rscrypto::{ Aes128GcmSiv, Aes128GcmSivKey, Aes128GcmSivTag, aead::{Nonce96, expert::AeadWithNonce}, @@ -28,18 +25,20 @@ fn assert_matches_oracle(key_bytes: &[u8; 16], nonce_bytes: &[u8; 12], aad: &[u8 let nonce = Nonce96::from_bytes(*nonce_bytes); let cipher = Aes128GcmSiv::new(&key); - let oracle = Oracle::new(GenericArray::from_slice(key_bytes)); - let oracle_nonce = GenericArray::from_slice(nonce_bytes); + let oracle = Oracle::new_from_slice(key_bytes).expect("AES-128-GCM-SIV oracle key length must be valid"); + let oracle_nonce = OracleNonce::from(*nonce_bytes); // Encrypt with rscrypto. let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto must seal valid AES-128-GCM-SIV oracle input"); // Encrypt with oracle. let mut oracle_buf = plaintext.to_vec(); let oracle_tag = oracle - .encrypt_in_place_detached(oracle_nonce, aad, &mut oracle_buf) - .unwrap(); + .encrypt_inout_detached(&oracle_nonce, aad, oracle_buf.as_mut_slice().into()) + .expect("RustCrypto must seal valid AES-128-GCM-SIV oracle input"); assert_eq!(ours, oracle_buf, "ciphertext mismatch (len={})", plaintext.len()); assert_eq!( @@ -50,7 +49,9 @@ fn assert_matches_oracle(key_bytes: &[u8; 16], nonce_bytes: &[u8; 12], aad: &[u8 ); // Decrypt with rscrypto. - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AES-128-GCM-SIV ciphertext must authenticate"); assert_eq!(ours, plaintext, "decrypt round-trip failed (len={})", plaintext.len()); } @@ -97,7 +98,7 @@ fn aes128gcmsiv_oracle_block_boundary_sizes() { fn aes128gcmsiv_oracle_large_input() { let key = [0x77u8; 16]; let nonce = [0x88u8; 12]; - let plaintext: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let plaintext: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, b"large", &plaintext); } @@ -109,8 +110,8 @@ fn aes128gcmsiv_oracle_aad_size_sweep() { // Sweeps cover the wide-POLYVAL 4-block boundary (64-byte chunks) and the // partial-tail seam at +/-1 around 16-byte block boundaries. - for aad_len in [0, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { - let aad: Vec = (0..aad_len).map(|i| (i & 0xFF) as u8).collect(); + for aad_len in [0usize, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { + let aad: Vec = (0..aad_len).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, &aad, plaintext); } } @@ -124,10 +125,14 @@ fn aes128gcmsiv_is_deterministic_under_nonce_reuse() { let plaintext = b"same nonce, same plaintext, same tag"; let mut first = plaintext.to_vec(); - let first_tag = cipher.encrypt_in_place(&nonce, aad, &mut first).unwrap(); + let first_tag = cipher + .encrypt_in_place(&nonce, aad, &mut first) + .expect("first deterministic AES-128-GCM-SIV seal must succeed"); let mut second = plaintext.to_vec(); - let second_tag = cipher.encrypt_in_place(&nonce, aad, &mut second).unwrap(); + let second_tag = cipher + .encrypt_in_place(&nonce, aad, &mut second) + .expect("second deterministic AES-128-GCM-SIV seal must succeed"); assert_eq!(first, second, "ciphertext changed across identical AES-GCM-SIV inputs"); assert_eq!( @@ -146,14 +151,15 @@ fn aes128gcmsiv_rejects_modified_tag() { let cipher = Aes128GcmSiv::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("AES-128-GCM-SIV tag-forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes128GcmSivTag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes128GcmSivTag::from_bytes(tag)) + .expect_err("AES-128-GCM-SIV must reject a modified tag"); } #[test] @@ -163,10 +169,14 @@ fn aes128gcmsiv_rejects_modified_ciphertext() { let cipher = Aes128GcmSiv::new(&key); let mut buffer = *b"tamper-detect"; - let tag = cipher.encrypt_in_place(&nonce, b"", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"", &mut buffer) + .expect("AES-128-GCM-SIV ciphertext-tampering fixture must seal"); buffer[0] ^= 1; - assert!(cipher.decrypt_in_place(&nonce, b"", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"", &mut buffer, &tag) + .expect_err("AES-128-GCM-SIV must reject modified ciphertext"); } #[test] @@ -176,7 +186,11 @@ fn aes128gcmsiv_rejects_wrong_aad() { let cipher = Aes128GcmSiv::new(&key); let mut buffer = *b"aad-mismatch"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buffer) + .expect("AES-128-GCM-SIV AAD-mismatch fixture must seal"); - assert!(cipher.decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag) + .expect_err("AES-128-GCM-SIV must reject incorrect associated data"); } diff --git a/tests/aes256gcm_oracle.rs b/tests/aes256gcm_oracle.rs index dedd58bf..c9bc834d 100644 --- a/tests/aes256gcm_oracle.rs +++ b/tests/aes256gcm_oracle.rs @@ -24,10 +24,10 @@ use rscrypto::{ fn deterministic_bytes(seed: u8, len: usize) -> Vec { let mut out = Vec::with_capacity(len); - let mut x = seed as u32; + let mut x = u32::from(seed); for _ in 0..len { x = x.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - out.push((x >> 24) as u8); + out.push(x.to_be_bytes()[0]); } out } @@ -42,13 +42,15 @@ fn assert_matches_oracle(key_bytes: &[u8; 32], nonce_bytes: &[u8; 12], aad: &[u8 // Encrypt with rscrypto. let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto must seal valid AES-256-GCM oracle input"); // Encrypt with oracle. let mut oracle_buf = plaintext.to_vec(); let oracle_tag = oracle .encrypt_inout_detached(&oracle_nonce, aad, oracle_buf.as_mut_slice().into()) - .unwrap(); + .expect("RustCrypto must seal valid AES-256-GCM oracle input"); assert_eq!(ours, oracle_buf, "ciphertext mismatch (len={})", plaintext.len()); assert_eq!( @@ -59,7 +61,9 @@ fn assert_matches_oracle(key_bytes: &[u8; 32], nonce_bytes: &[u8; 12], aad: &[u8 ); // Decrypt with rscrypto. - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AES-256-GCM ciphertext must authenticate"); assert_eq!(ours, plaintext, "decrypt round-trip failed (len={})", plaintext.len()); } @@ -108,7 +112,7 @@ fn aes256gcm_oracle_large_input() { let key = [0x77u8; 32]; let nonce = [0x88u8; 12]; // 8 KiB — exercises multi-block GHASH and CTR paths. - let plaintext: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let plaintext: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, b"large", &plaintext); } @@ -120,8 +124,8 @@ fn aes256gcm_oracle_aad_size_sweep() { // Sweeps cover the wide-GHASH 4-block boundary (64-byte chunks) and the // partial-tail seam at +/-1 around 16-byte block boundaries. - for aad_len in [0, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { - let aad: Vec = (0..aad_len).map(|i| (i & 0xFF) as u8).collect(); + for aad_len in [0usize, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { + let aad: Vec = (0..aad_len).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, &aad, plaintext); } } @@ -132,13 +136,15 @@ fn aes256gcm_oracle_all_short_lengths() { for size in 0..=255usize { let mut nonce = [0x28u8; 12]; - nonce[8..12].copy_from_slice(&(size as u32).to_be_bytes()); + let size_word = u32::try_from(size).expect("short-length case must fit in a 32-bit nonce field"); + let size_byte = size.to_le_bytes()[0]; + nonce[8..12].copy_from_slice(&size_word.to_be_bytes()); let aad = if size % 3 == 0 { Vec::new() } else { - deterministic_bytes(0x52 ^ size as u8, size % 97) + deterministic_bytes(0x52 ^ size_byte, size % 97) }; - let plaintext = deterministic_bytes(0x91 ^ size as u8, size); + let plaintext = deterministic_bytes(0x91 ^ size_byte, size); assert_matches_oracle(&key, &nonce, &aad, &plaintext); } } @@ -150,9 +156,11 @@ fn aes256gcm_oracle_large_aligned_and_unaligned_lengths() { for (case_idx, &size) in [4096usize, 4097, 8191, 8192, 8193, 16_384, 16_385].iter().enumerate() { let mut nonce = [0x81u8; 12]; - nonce[4..8].copy_from_slice(&(case_idx as u32).to_be_bytes()); - nonce[8..12].copy_from_slice(&(size as u32).to_be_bytes()); - let plaintext = deterministic_bytes(0x43 ^ case_idx as u8, size); + let case_word = u32::try_from(case_idx).expect("GCM oracle case index must fit in a 32-bit nonce field"); + let size_word = u32::try_from(size).expect("GCM oracle size must fit in a 32-bit nonce field"); + nonce[4..8].copy_from_slice(&case_word.to_be_bytes()); + nonce[8..12].copy_from_slice(&size_word.to_be_bytes()); + let plaintext = deterministic_bytes(0x43 ^ case_idx.to_le_bytes()[0], size); for aad in &aad_cases { assert_matches_oracle(&key, &nonce, aad, &plaintext); } @@ -168,14 +176,15 @@ fn aes256gcm_rejects_modified_tag() { let cipher = Aes256Gcm::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("AES-256-GCM tag-forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes256GcmTag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes256GcmTag::from_bytes(tag)) + .expect_err("AES-256-GCM must reject a modified tag"); } #[test] @@ -185,10 +194,14 @@ fn aes256gcm_rejects_modified_ciphertext() { let cipher = Aes256Gcm::new(&key); let mut buffer = *b"tamper-detect"; - let tag = cipher.encrypt_in_place(&nonce, b"", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"", &mut buffer) + .expect("AES-256-GCM ciphertext-tampering fixture must seal"); buffer[0] ^= 1; - assert!(cipher.decrypt_in_place(&nonce, b"", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"", &mut buffer, &tag) + .expect_err("AES-256-GCM must reject modified ciphertext"); } #[test] @@ -198,7 +211,11 @@ fn aes256gcm_rejects_wrong_aad() { let cipher = Aes256Gcm::new(&key); let mut buffer = *b"aad-mismatch"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buffer) + .expect("AES-256-GCM AAD-mismatch fixture must seal"); - assert!(cipher.decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag) + .expect_err("AES-256-GCM must reject incorrect associated data"); } diff --git a/tests/aes256gcmsiv_oracle.rs b/tests/aes256gcmsiv_oracle.rs index cdf3bb03..ee084554 100644 --- a/tests/aes256gcmsiv_oracle.rs +++ b/tests/aes256gcmsiv_oracle.rs @@ -13,10 +13,7 @@ #![cfg(feature = "aead")] -use aes_gcm_siv::{ - Aes256GcmSiv as Oracle, KeyInit, - aead::{AeadInPlace, generic_array::GenericArray}, -}; +use aes_gcm_siv::{Aes256GcmSiv as Oracle, KeyInit, Nonce as OracleNonce, aead::AeadInOut}; use rscrypto::{ Aes256GcmSiv, Aes256GcmSivKey, Aes256GcmSivTag, aead::{Nonce96, expert::AeadWithNonce}, @@ -27,18 +24,20 @@ fn assert_matches_oracle(key_bytes: &[u8; 32], nonce_bytes: &[u8; 12], aad: &[u8 let nonce = Nonce96::from_bytes(*nonce_bytes); let cipher = Aes256GcmSiv::new(&key); - let oracle = Oracle::new(GenericArray::from_slice(key_bytes)); - let oracle_nonce = GenericArray::from_slice(nonce_bytes); + let oracle = Oracle::new_from_slice(key_bytes).expect("AES-256-GCM-SIV oracle key length must be valid"); + let oracle_nonce = OracleNonce::from(*nonce_bytes); // Encrypt with rscrypto. let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto must seal valid AES-256-GCM-SIV oracle input"); // Encrypt with oracle. let mut oracle_buf = plaintext.to_vec(); let oracle_tag = oracle - .encrypt_in_place_detached(oracle_nonce, aad, &mut oracle_buf) - .unwrap(); + .encrypt_inout_detached(&oracle_nonce, aad, oracle_buf.as_mut_slice().into()) + .expect("RustCrypto must seal valid AES-256-GCM-SIV oracle input"); assert_eq!(ours, oracle_buf, "ciphertext mismatch (len={})", plaintext.len()); assert_eq!( @@ -49,7 +48,9 @@ fn assert_matches_oracle(key_bytes: &[u8; 32], nonce_bytes: &[u8; 12], aad: &[u8 ); // Decrypt with rscrypto. - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AES-256-GCM-SIV ciphertext must authenticate"); assert_eq!(ours, plaintext, "decrypt round-trip failed (len={})", plaintext.len()); } @@ -96,7 +97,7 @@ fn aes256gcmsiv_oracle_block_boundary_sizes() { fn aes256gcmsiv_oracle_large_input() { let key = [0x77u8; 32]; let nonce = [0x88u8; 12]; - let plaintext: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let plaintext: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, b"large", &plaintext); } @@ -108,8 +109,8 @@ fn aes256gcmsiv_oracle_aad_size_sweep() { // Sweeps cover the wide-POLYVAL 4-block boundary (64-byte chunks) and the // partial-tail seam at +/-1 around 16-byte block boundaries. - for aad_len in [0, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { - let aad: Vec = (0..aad_len).map(|i| (i & 0xFF) as u8).collect(); + for aad_len in [0usize, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { + let aad: Vec = (0..aad_len).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, &aad, plaintext); } } @@ -123,10 +124,14 @@ fn aes256gcmsiv_is_deterministic_under_nonce_reuse() { let plaintext = b"same nonce, same plaintext, same tag"; let mut first = plaintext.to_vec(); - let first_tag = cipher.encrypt_in_place(&nonce, aad, &mut first).unwrap(); + let first_tag = cipher + .encrypt_in_place(&nonce, aad, &mut first) + .expect("first deterministic AES-256-GCM-SIV seal must succeed"); let mut second = plaintext.to_vec(); - let second_tag = cipher.encrypt_in_place(&nonce, aad, &mut second).unwrap(); + let second_tag = cipher + .encrypt_in_place(&nonce, aad, &mut second) + .expect("second deterministic AES-256-GCM-SIV seal must succeed"); assert_eq!(first, second, "ciphertext changed across identical AES-GCM-SIV inputs"); assert_eq!( @@ -145,14 +150,15 @@ fn aes256gcmsiv_rejects_modified_tag() { let cipher = Aes256GcmSiv::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("AES-256-GCM-SIV tag-forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes256GcmSivTag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes256GcmSivTag::from_bytes(tag)) + .expect_err("AES-256-GCM-SIV must reject a modified tag"); } #[test] @@ -162,10 +168,14 @@ fn aes256gcmsiv_rejects_modified_ciphertext() { let cipher = Aes256GcmSiv::new(&key); let mut buffer = *b"tamper-detect"; - let tag = cipher.encrypt_in_place(&nonce, b"", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"", &mut buffer) + .expect("AES-256-GCM-SIV ciphertext-tampering fixture must seal"); buffer[0] ^= 1; - assert!(cipher.decrypt_in_place(&nonce, b"", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"", &mut buffer, &tag) + .expect_err("AES-256-GCM-SIV must reject modified ciphertext"); } #[test] @@ -175,7 +185,11 @@ fn aes256gcmsiv_rejects_wrong_aad() { let cipher = Aes256GcmSiv::new(&key); let mut buffer = *b"aad-mismatch"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buffer) + .expect("AES-256-GCM-SIV AAD-mismatch fixture must seal"); - assert!(cipher.decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag) + .expect_err("AES-256-GCM-SIV must reject incorrect associated data"); } diff --git a/tests/aes_gcm_aarch64_asm_oracle.rs b/tests/aes_gcm_aarch64_asm_oracle.rs index be3c0c86..ba431e80 100644 --- a/tests/aes_gcm_aarch64_asm_oracle.rs +++ b/tests/aes_gcm_aarch64_asm_oracle.rs @@ -27,20 +27,24 @@ fn deterministic_bytes(seed: u8, len: usize) -> Vec { fn assert_aes128_matches_oracle(len: usize, aad: &[u8]) { let key_bytes = [0x31u8; 16]; let mut nonce_bytes = [0x42u8; 12]; - nonce_bytes[4..8].copy_from_slice(&(aad.len() as u32).to_be_bytes()); - nonce_bytes[8..12].copy_from_slice(&(len as u32).to_be_bytes()); - let plaintext = deterministic_bytes(0x80 ^ len as u8, len); + let aad_len = u32::try_from(aad.len()).expect("oracle AAD length must fit the 32-bit nonce field"); + let text_len = u32::try_from(len).expect("oracle text length must fit the 32-bit nonce field"); + nonce_bytes[4..8].copy_from_slice(&aad_len.to_be_bytes()); + nonce_bytes[8..12].copy_from_slice(&text_len.to_be_bytes()); + let plaintext = deterministic_bytes(0x80 ^ len.to_le_bytes()[0], len); let cipher = Aes128Gcm::new(&Aes128GcmKey::from_bytes(key_bytes)); let nonce = Nonce96::from_bytes(nonce_bytes); let mut ours = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("bounded AES-128-GCM oracle input must seal"); let oracle = Aes128Oracle::new(&Array(key_bytes)); let mut expected = plaintext.clone(); let expected_tag = oracle .encrypt_inout_detached(&Array(nonce_bytes), aad, expected.as_mut_slice().into()) - .unwrap(); + .expect("bounded AES-128-GCM oracle input must seal"); assert_eq!(ours, expected, "AES-128-GCM ciphertext mismatch at len {len}"); assert_eq!( @@ -49,27 +53,33 @@ fn assert_aes128_matches_oracle(len: usize, aad: &[u8]) { "AES-128-GCM tag mismatch at len {len}" ); - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AES-128-GCM ciphertext and tag must open"); assert_eq!(ours, plaintext, "AES-128-GCM open mismatch at len {len}"); } fn assert_aes256_matches_oracle(len: usize, aad: &[u8]) { let key_bytes = [0x53u8; 32]; let mut nonce_bytes = [0x64u8; 12]; - nonce_bytes[4..8].copy_from_slice(&(aad.len() as u32).to_be_bytes()); - nonce_bytes[8..12].copy_from_slice(&(len as u32).to_be_bytes()); - let plaintext = deterministic_bytes(0xA0 ^ len as u8, len); + let aad_len = u32::try_from(aad.len()).expect("oracle AAD length must fit the 32-bit nonce field"); + let text_len = u32::try_from(len).expect("oracle text length must fit the 32-bit nonce field"); + nonce_bytes[4..8].copy_from_slice(&aad_len.to_be_bytes()); + nonce_bytes[8..12].copy_from_slice(&text_len.to_be_bytes()); + let plaintext = deterministic_bytes(0xA0 ^ len.to_le_bytes()[0], len); let cipher = Aes256Gcm::new(&Aes256GcmKey::from_bytes(key_bytes)); let nonce = Nonce96::from_bytes(nonce_bytes); let mut ours = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("bounded AES-256-GCM oracle input must seal"); let oracle = Aes256Oracle::new(&Array(key_bytes)); let mut expected = plaintext.clone(); let expected_tag = oracle .encrypt_inout_detached(&Array(nonce_bytes), aad, expected.as_mut_slice().into()) - .unwrap(); + .expect("bounded AES-256-GCM oracle input must seal"); assert_eq!(ours, expected, "AES-256-GCM ciphertext mismatch at len {len}"); assert_eq!( @@ -78,7 +88,9 @@ fn assert_aes256_matches_oracle(len: usize, aad: &[u8]) { "AES-256-GCM tag mismatch at len {len}" ); - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AES-256-GCM ciphertext and tag must open"); assert_eq!(ours, plaintext, "AES-256-GCM open mismatch at len {len}"); } diff --git a/tests/api_consistency.rs b/tests/api_consistency.rs index 28c78d41..15aed348 100644 --- a/tests/api_consistency.rs +++ b/tests/api_consistency.rs @@ -1,12 +1,23 @@ -#![allow(unused_imports)] - #[cfg(all(feature = "checksums", feature = "std"))] -use std::io::{Cursor, Read, Write}; +use std::io::Write; #[cfg(any(feature = "ecdsa-p256", feature = "ecdsa-p384"))] use rscrypto::EcdsaKeyGenerationError; +use rscrypto::Kem; #[cfg(any(feature = "hmac", feature = "hmac-sha3"))] use rscrypto::Mac; +#[cfg(any( + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + all(feature = "rsa", feature = "getrandom") +))] +use rscrypto::TrySigner; +#[cfg(all(feature = "rsa", feature = "getrandom"))] +use rscrypto::TrySignerInto; +use rscrypto::VerificationError; +#[cfg(any(feature = "ecdsa-p256", feature = "ecdsa-p384", feature = "ed25519", feature = "rsa"))] +use rscrypto::Verifier; #[cfg(feature = "aead")] use rscrypto::aead::expert::AeadWithNonce; #[cfg(feature = "aead")] @@ -24,7 +35,7 @@ use rscrypto::{ #[cfg(feature = "ecdsa-p256")] use rscrypto::{EcdsaP256Keypair, EcdsaP256PublicKey, EcdsaP256SecretKey}; #[cfg(feature = "ecdsa-p384")] -use rscrypto::{EcdsaP384Keypair, EcdsaP384PublicKey, EcdsaP384SecretKey}; +use rscrypto::{EcdsaP384Keypair, EcdsaP384PublicKey}; #[cfg(feature = "ed25519")] use rscrypto::{Ed25519Keypair, Ed25519PublicKey, Ed25519SecretKey}; #[cfg(feature = "hkdf")] @@ -33,7 +44,6 @@ use rscrypto::{HkdfSha256, HkdfSha384, HkdfSha512, auth::HkdfOutputLengthError}; use rscrypto::{HmacSha3_224, HmacSha3_256, HmacSha3_384, HmacSha3_512}; #[cfg(feature = "hmac")] use rscrypto::{HmacSha256, HmacSha384, HmacSha512}; -use rscrypto::{Kem, TrySigner, TrySignerInto, VerificationError, Verifier}; #[cfg(feature = "kmac")] use rscrypto::{Kmac128, Kmac256}; #[cfg(feature = "poly1305")] @@ -123,7 +133,9 @@ fn assert_mac_api() { let expected = mac.finalize(); mac.reset(); mac.update(b"abc"); - assert!(mac.verify(&expected).is_ok()); + mac + .verify(&expected) + .expect("MAC must verify its freshly finalized tag"); } #[cfg(any(feature = "hmac", feature = "hmac-sha3"))] @@ -174,27 +186,41 @@ where let plaintext = b"abc"; let mut sealed = [0u8; 19]; - aead.encrypt(&nonce, b"aad", plaintext, &mut sealed).unwrap(); + aead + .encrypt(&nonce, b"aad", plaintext, &mut sealed) + .expect("fixed-size AEAD seal buffer must fit plaintext and tag"); let mut opened = [0u8; 3]; - aead.decrypt(&nonce, b"aad", &sealed, &mut opened).unwrap(); + aead + .decrypt(&nonce, b"aad", &sealed, &mut opened) + .expect("freshly sealed AEAD ciphertext must authenticate"); assert_eq!(&opened, plaintext); #[cfg(feature = "alloc")] { - let sealed_vec = aead.encrypt_to_vec(&nonce, b"aad", plaintext).unwrap(); + let sealed_vec = aead + .encrypt_to_vec(&nonce, b"aad", plaintext) + .expect("AEAD vector allocation must accept a three-byte plaintext"); assert_eq!(sealed_vec.as_slice(), sealed); - let opened_vec = aead.decrypt_to_vec(&nonce, b"aad", &sealed_vec).unwrap(); + let opened_vec = aead + .decrypt_to_vec(&nonce, b"aad", &sealed_vec) + .expect("freshly sealed AEAD vector must authenticate"); assert_eq!(opened_vec.as_slice(), plaintext); let mut tampered = sealed_vec; tampered[0] ^= 1; - assert!(aead.decrypt_to_vec(&nonce, b"aad", &tampered).is_err()); + aead + .decrypt_to_vec(&nonce, b"aad", &tampered) + .expect_err("tampered AEAD ciphertext must fail authentication"); } let mut detached = *b"abc"; - let tag = aead.encrypt_in_place_detached(&nonce, b"aad", &mut detached).unwrap(); - aead.decrypt_in_place(&nonce, b"aad", &mut detached, &tag).unwrap(); + let tag = aead + .encrypt_in_place_detached(&nonce, b"aad", &mut detached) + .expect("detached AEAD seal must accept a three-byte plaintext"); + aead + .decrypt_in_place(&nonce, b"aad", &mut detached, &tag) + .expect("freshly sealed detached AEAD ciphertext must authenticate"); assert_eq!(&detached, plaintext); } @@ -248,7 +274,7 @@ fn all_xofs_follow_new_update_finalize_xof_and_xof() { cshake.update(data); assert_eq!(streaming, squeeze_32(cshake.finalize_xof())); - let mut cxof = AsconCxof128::new(b"ctx=v1").unwrap(); + let mut cxof = AsconCxof128::new(b"ctx=v1").expect("short Ascon-CXOF customization must be valid"); cxof.update(data); let streaming = squeeze_32(cxof.finalize_xof()); cxof.reset(); @@ -292,7 +318,9 @@ fn kmac_follows_new_update_finalize_into_reset_and_verify() { kmac128.update(b"abc"); kmac128.finalize_into(&mut actual128); assert_eq!(actual128, expected128); - assert!(kmac128.verify(&expected128).is_ok()); + kmac128 + .verify(&expected128) + .expect("KMAC128 must verify its freshly finalized tag"); let mut kmac = Kmac256::new(b"api-consistency-key", b"ctx=v1"); kmac.update(b"abc"); @@ -304,7 +332,9 @@ fn kmac_follows_new_update_finalize_into_reset_and_verify() { kmac.update(b"abc"); kmac.finalize_into(&mut actual); assert_eq!(actual, expected); - assert!(kmac.verify(&expected).is_ok()); + kmac + .verify(&expected) + .expect("KMAC256 must verify its freshly finalized tag"); } #[test] @@ -314,32 +344,38 @@ fn hkdfs_follow_new_expand_and_derive_array_conventions() { let hkdf384 = HkdfSha384::new(b"salt", b"ikm"); let hkdf512 = HkdfSha512::new(b"salt", b"ikm"); - let okm256 = hkdf256.expand_array::<32>(b"info").unwrap(); - let okm384 = hkdf384.expand_array::<48>(b"info").unwrap(); - let okm512 = hkdf512.expand_array::<64>(b"info").unwrap(); + let okm256 = hkdf256 + .expand_array::<32>(b"info") + .expect("32-byte HKDF-SHA-256 output must be valid"); + let okm384 = hkdf384 + .expand_array::<48>(b"info") + .expect("48-byte HKDF-SHA-384 output must be valid"); + let okm512 = hkdf512 + .expand_array::<64>(b"info") + .expect("64-byte HKDF-SHA-512 output must be valid"); assert_eq!( okm256, - HkdfSha256::derive_array::<32>(b"salt", b"ikm", b"info").unwrap() + HkdfSha256::derive_array::<32>(b"salt", b"ikm", b"info").expect("32-byte HKDF-SHA-256 derivation must be valid") ); assert_eq!( okm384, - HkdfSha384::derive_array::<48>(b"salt", b"ikm", b"info").unwrap() + HkdfSha384::derive_array::<48>(b"salt", b"ikm", b"info").expect("48-byte HKDF-SHA-384 derivation must be valid") ); assert_eq!( okm512, - HkdfSha512::derive_array::<64>(b"salt", b"ikm", b"info").unwrap() + HkdfSha512::derive_array::<64>(b"salt", b"ikm", b"info").expect("64-byte HKDF-SHA-512 derivation must be valid") ); assert_eq!( - HkdfSha256::derive_array::<32>(b"salt", b"ikm", b"info").unwrap(), + HkdfSha256::derive_array::<32>(b"salt", b"ikm", b"info").expect("32-byte HKDF-SHA-256 derivation must be valid"), okm256 ); assert_eq!( - HkdfSha384::derive_array::<48>(b"salt", b"ikm", b"info").unwrap(), + HkdfSha384::derive_array::<48>(b"salt", b"ikm", b"info").expect("48-byte HKDF-SHA-384 derivation must be valid"), okm384 ); assert_eq!( - HkdfSha512::derive_array::<64>(b"salt", b"ikm", b"info").unwrap(), + HkdfSha512::derive_array::<64>(b"salt", b"ikm", b"info").expect("64-byte HKDF-SHA-512 derivation must be valid"), okm512 ); assert_eq!(hkdf256.prk().len(), 32); @@ -361,7 +397,8 @@ fn poly1305_consumes_one_time_key_and_verifies() { assert_eq!(tag.as_bytes().len(), Poly1305Tag::LENGTH); let key = Poly1305OneTimeKey::from_bytes([0x42; Poly1305OneTimeKey::LENGTH]); - assert!(Poly1305::verify_once(key, b"api-consistency-poly1305", &tag).is_ok()); + Poly1305::verify_once(key, b"api-consistency-poly1305", &tag) + .expect("Poly1305 must verify its freshly generated tag"); } #[test] @@ -371,17 +408,21 @@ fn ed25519_types_follow_byte_roundtrip_and_verify_conventions() { out.fill(0x24); Ok::<(), ()>(()) }) - .unwrap(); + .expect("deterministic Ed25519 entropy callback must generate a key"); let keypair = Ed25519Keypair::from_secret_key(secret.duplicate_secret()); let public = Ed25519PublicKey::from_bytes(keypair.public_key().to_bytes()); - let signature = TrySigner::try_sign(&keypair, b"api-consistency-ed25519").unwrap(); + let signature = TrySigner::try_sign(&keypair, b"api-consistency-ed25519") + .expect("Ed25519 signing with a valid generated key must succeed"); assert_eq!(*secret.expose_secret().as_bytes(), *secret.as_bytes()); assert_eq!(secret.duplicate_secret().as_bytes(), secret.as_bytes()); assert_eq!(public.to_bytes(), *public.as_bytes()); assert_eq!(signature.to_bytes(), *signature.as_bytes()); - assert!(public.verify(b"api-consistency-ed25519", &signature).is_ok()); - assert!(Verifier::verify(&public, b"api-consistency-ed25519", &signature).is_ok()); + public + .verify(b"api-consistency-ed25519", &signature) + .expect("Ed25519 public-key method must verify the fresh signature"); + Verifier::verify(&public, b"api-consistency-ed25519", &signature) + .expect("Ed25519 Verifier trait must verify the fresh signature"); } #[test] @@ -391,9 +432,11 @@ fn x25519_types_follow_byte_roundtrip_conventions() { out.fill(0x42); Ok::<(), ()>(()) }) - .unwrap(); + .expect("deterministic X25519 entropy callback must generate a key"); let public = X25519PublicKey::from_bytes(secret.public_key().to_bytes()); - let shared = secret.diffie_hellman(&public).unwrap(); + let shared = secret + .diffie_hellman(&public) + .expect("X25519 with a valid public key must produce a nonzero shared secret"); assert_eq!(*secret.expose_secret().as_bytes(), *secret.as_bytes()); assert_eq!(public.to_bytes(), *public.as_bytes()); @@ -407,20 +450,24 @@ fn ecdsa_p256_keygen_and_native_signature_traits_are_consistent() { out.fill(1); Ok::<(), ()>(()) }) - .unwrap(); - let public = EcdsaP256PublicKey::from_sec1_bytes(&keypair.public_key().to_sec1_bytes()).unwrap(); - let signature = TrySigner::try_sign(&keypair, b"api-consistency-ecdsa-p256").unwrap(); + .expect("deterministic P-256 entropy callback must generate a key"); + let public = EcdsaP256PublicKey::from_sec1_bytes(&keypair.public_key().to_sec1_bytes()) + .expect("generated P-256 public key must round-trip through SEC1"); + let signature = TrySigner::try_sign(&keypair, b"api-consistency-ecdsa-p256") + .expect("P-256 signing with a valid generated key must succeed"); - assert!(Verifier::verify(&public, b"api-consistency-ecdsa-p256", &signature).is_ok()); + Verifier::verify(&public, b"api-consistency-ecdsa-p256", &signature) + .expect("P-256 Verifier trait must verify the fresh signature"); let rejected = EcdsaP256SecretKey::try_generate_with(|out| { out.fill(0); Ok::<(), ()>(()) }) - .unwrap_err(); + .expect_err("an all-zero P-256 secret key must be rejected"); assert_eq!(rejected, EcdsaKeyGenerationError::InvalidSecretKey); - let rng_error = EcdsaP256SecretKey::try_generate_with(|_| Err("rng")).unwrap_err(); + let rng_error = EcdsaP256SecretKey::try_generate_with(|_| Err("rng")) + .expect_err("P-256 key generation must preserve entropy-source failure"); assert_eq!(rng_error, EcdsaKeyGenerationError::Random("rng")); } @@ -431,26 +478,35 @@ fn ecdsa_p384_keygen_and_native_signature_traits_are_consistent() { out.fill(1); Ok::<(), ()>(()) }) - .unwrap(); - let public = EcdsaP384PublicKey::from_sec1_bytes(&keypair.public_key().to_sec1_bytes()).unwrap(); - let signature = TrySigner::try_sign(&keypair, b"api-consistency-ecdsa-p384").unwrap(); - - assert!(Verifier::verify(&public, b"api-consistency-ecdsa-p384", &signature).is_ok()); + .expect("deterministic P-384 entropy callback must generate a key"); + let public = EcdsaP384PublicKey::from_sec1_bytes(&keypair.public_key().to_sec1_bytes()) + .expect("generated P-384 public key must round-trip through SEC1"); + let signature = TrySigner::try_sign(&keypair, b"api-consistency-ecdsa-p384") + .expect("P-384 signing with a valid generated key must succeed"); + + Verifier::verify(&public, b"api-consistency-ecdsa-p384", &signature) + .expect("P-384 Verifier trait must verify the fresh signature"); } #[test] #[cfg(feature = "rsa")] fn rsa_signature_verifier_requires_a_bound_profile() { - let key = RsaPublicKey::from_spki_der(include_bytes!("../benches/rsa_fixtures/rsa3072_spki.der")).unwrap(); + let key = RsaPublicKey::from_spki_der(include_bytes!("../benches/rsa_fixtures/rsa3072_spki.der")) + .expect("embedded RSA-3072 SPKI fixture must parse"); let message = b"rscrypto RSA-PSS verification fixture"; let pss_signature = include_bytes!("../benches/rsa_fixtures/rsa3072_pss_sha256.sig"); let pss_verifier = key.verifier(RsaSignatureProfile::pss(RsaPssProfile::Sha256)); - assert!(pss_verifier.verify(message, pss_signature).is_ok()); - assert!(Verifier::verify(&pss_verifier, message, pss_signature.as_slice()).is_ok()); + pss_verifier + .verify(message, pss_signature) + .expect("profile-bound RSA verifier must accept the matching PSS fixture"); + Verifier::verify(&pss_verifier, message, pss_signature.as_slice()) + .expect("RSA Verifier trait must accept the matching PSS fixture"); let wrong_profile = key.verifier(RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256)); - assert!(wrong_profile.verify(message, pss_signature).is_err()); + wrong_profile + .verify(message, pss_signature) + .expect_err("RSA verifier must reject a PSS signature under the PKCS1-v1_5 profile"); } #[test] @@ -483,7 +539,7 @@ fn kem_trait_uses_typed_key_ciphertext_and_secret_outputs() { out.copy_from_slice(&[1, 2, 3, 4]); Ok(()) }) - .unwrap(); + .expect("infallible ToyKem key generation must succeed"); assert_eq!(encapsulation_key.as_ref(), &[1, 2, 3]); assert_eq!(decapsulation_key.as_ref(), &[1, 2, 3, 4]); @@ -491,11 +547,12 @@ fn kem_trait_uses_typed_key_ciphertext_and_secret_outputs() { out.copy_from_slice(&[8, 9]); Ok(()) }) - .unwrap(); + .expect("infallible ToyKem encapsulation must succeed"); assert_eq!(ciphertext.as_ref(), &[1, 2, 3, 8, 9]); assert_eq!(encapsulated_secret.as_ref(), &[8, 9]); - let decapsulated_secret = ToyKem::decapsulate(&decapsulation_key, &ciphertext).unwrap(); + let decapsulated_secret = + ToyKem::decapsulate(&decapsulation_key, &ciphertext).expect("infallible ToyKem decapsulation must succeed"); assert_eq!(decapsulated_secret.as_ref(), &[4, 9]); } @@ -576,24 +633,30 @@ fn all_aeads_follow_new_encrypt_decrypt_and_detached_aliases() { #[cfg(all(feature = "aead", feature = "getrandom", feature = "alloc"))] fn aead_random_to_vec_seals_and_opens() { let cipher = ChaCha20Poly1305::new(&ChaCha20Poly1305Key::from_bytes([0x33; ChaCha20Poly1305::KEY_SIZE])); - let (nonce, sealed) = cipher.seal_random_to_vec(b"aad", b"plaintext").unwrap(); - let opened = cipher.decrypt_to_vec(&nonce, b"aad", &sealed).unwrap(); + let (nonce, sealed) = cipher + .seal_random_to_vec(b"aad", b"plaintext") + .expect("random-nonce AEAD sealing must succeed"); + let opened = cipher + .decrypt_to_vec(&nonce, b"aad", &sealed) + .expect("freshly sealed random-nonce ciphertext must authenticate"); assert_eq!(opened, b"plaintext"); } #[test] #[cfg(all(feature = "checksums", feature = "std"))] -fn checksum_adapters_use_checksum() -> std::io::Result<()> { +fn checksum_adapters_use_checksum() { use rscrypto::{Checksum as _, Crc32C}; - let mut reader = Crc32C::reader(Cursor::new(b"abc".to_vec())); - std::io::copy(&mut reader, &mut std::io::sink())?; + let mut reader = Crc32C::reader(&b"abc"[..]); + let mut copied = Vec::new(); + std::io::copy(&mut reader, &mut copied).expect("checksum reader must stream into a Vec"); + assert_eq!(copied, b"abc"); assert_eq!(reader.checksum(), Crc32C::checksum(b"abc")); let mut writer = Crc32C::writer(Vec::new()); - writer.write_all(b"abc")?; + writer + .write_all(b"abc") + .expect("Vec-backed checksum writer must accept three bytes"); assert_eq!(writer.checksum(), Crc32C::checksum(b"abc")); - - Ok(()) } diff --git a/tests/argon2_differential.rs b/tests/argon2_differential.rs index 9c63baa9..85801505 100644 --- a/tests/argon2_differential.rs +++ b/tests/argon2_differential.rs @@ -34,31 +34,34 @@ fn oracle_hash( p: u32, out_len: usize, ) -> Vec { - let params = argon2::Params::new(m_kib, t, p, Some(out_len)).unwrap(); + let params = argon2::Params::new(m_kib, t, p, Some(out_len)) + .expect("RustCrypto must accept generated Argon2 differential parameters"); let ctx = argon2::Argon2::new(algo, argon2::Version::V0x13, params); let mut out = vec![0u8; out_len]; - ctx.hash_password_into(password, salt, &mut out).unwrap(); + ctx + .hash_password_into(password, salt, &mut out) + .expect("RustCrypto Argon2 differential derivation must succeed"); out } fn rs_hash_id(password: &[u8], salt: &[u8], m_kib: u32, t: u32, p: u32, out_len: usize) -> Vec { - let params = Argon2Params::new(m_kib, t, p).unwrap(); + let params = Argon2Params::new(m_kib, t, p).expect("generated Argon2id parameters must be valid"); let mut out = vec![0u8; out_len]; - Argon2id::derive(¶ms, password, salt, &mut out).unwrap(); + Argon2id::derive(¶ms, password, salt, &mut out).expect("Argon2id differential derivation must succeed"); out } fn rs_hash_d(password: &[u8], salt: &[u8], m_kib: u32, t: u32, p: u32, out_len: usize) -> Vec { - let params = Argon2Params::new(m_kib, t, p).unwrap(); + let params = Argon2Params::new(m_kib, t, p).expect("generated Argon2d parameters must be valid"); let mut out = vec![0u8; out_len]; - Argon2d::derive(¶ms, password, salt, &mut out).unwrap(); + Argon2d::derive(¶ms, password, salt, &mut out).expect("Argon2d differential derivation must succeed"); out } fn rs_hash_i(password: &[u8], salt: &[u8], m_kib: u32, t: u32, p: u32, out_len: usize) -> Vec { - let params = Argon2Params::new(m_kib, t, p).unwrap(); + let params = Argon2Params::new(m_kib, t, p).expect("generated Argon2i parameters must be valid"); let mut out = vec![0u8; out_len]; - Argon2i::derive(¶ms, password, salt, &mut out).unwrap(); + Argon2i::derive(¶ms, password, salt, &mut out).expect("Argon2i differential derivation must succeed"); out } @@ -74,7 +77,7 @@ proptest! { p in 1u32..=2, out_len in proptest::sample::select(vec![16usize, 32, 48]), ) { - prop_assume!(m >= 8 * p); + prop_assume!(m >= p.strict_mul(8)); let actual = rs_hash_id(&password, &salt, m, t, p, out_len); let expected = oracle_hash(argon2::Algorithm::Argon2id, &password, &salt, m, t, p, out_len); prop_assert_eq!(actual, expected); @@ -89,7 +92,7 @@ proptest! { p in 1u32..=2, out_len in proptest::sample::select(vec![16usize, 32, 48]), ) { - prop_assume!(m >= 8 * p); + prop_assume!(m >= p.strict_mul(8)); let actual = rs_hash_d(&password, &salt, m, t, p, out_len); let expected = oracle_hash(argon2::Algorithm::Argon2d, &password, &salt, m, t, p, out_len); prop_assert_eq!(actual, expected); @@ -104,7 +107,7 @@ proptest! { p in 1u32..=2, out_len in proptest::sample::select(vec![16usize, 32, 48]), ) { - prop_assume!(m >= 8 * p); + prop_assume!(m >= p.strict_mul(8)); let actual = rs_hash_i(&password, &salt, m, t, p, out_len); let expected = oracle_hash(argon2::Algorithm::Argon2i, &password, &salt, m, t, p, out_len); prop_assert_eq!(actual, expected); @@ -118,12 +121,14 @@ proptest! { t in 1u32..=2, p in 1u32..=2, ) { - prop_assume!(m >= 8 * p); + prop_assume!(m >= p.strict_mul(8)); let params = Argon2Params::new(m, t, p) - .unwrap(); + .expect("generated Argon2 verification parameters must be valid"); let mut hash = [0u8; 32]; - Argon2id::derive(¶ms, &password, &salt, &mut hash).unwrap(); - prop_assert!(Argon2id::verify(¶ms, &password, &salt, &hash).is_ok()); + Argon2id::derive(¶ms, &password, &salt, &mut hash) + .expect("Argon2id verification fixture must derive"); + Argon2id::verify(¶ms, &password, &salt, &hash) + .expect("fresh Argon2id property-test hash must verify"); } } @@ -173,45 +178,43 @@ fn argon2id_p8_matches_oracle() { /// — see the doc comment on `verify` in `src/auth/argon2/mod.rs`. #[test] fn argon2id_verify_rejects_length_mismatch() { - let params = Argon2Params::new(8, 1, 1).unwrap(); + let params = Argon2Params::new(8, 1, 1).expect("length-mismatch test parameters must be valid"); let password = b"pw"; let salt = b"saltsalt"; let mut hash = [0u8; 32]; - Argon2id::derive(¶ms, password, salt, &mut hash).unwrap(); + Argon2id::derive(¶ms, password, salt, &mut hash).expect("length-mismatch fixture must derive"); // Sanity: correct length verifies. - assert!(Argon2id::verify(¶ms, password, salt, &hash).is_ok()); + Argon2id::verify(¶ms, password, salt, &hash).expect("fresh Argon2id hash must verify"); // Various wrong lengths: shorter, longer, empty. let too_short: &[u8] = &hash[..16]; let too_long: &[u8] = &[hash.as_ref(), &[0u8; 8]].concat()[..40]; let empty: &[u8] = &[]; - assert!(Argon2id::verify(¶ms, password, salt, too_short).is_err()); - assert!(Argon2id::verify(¶ms, password, salt, too_long).is_err()); - assert!(Argon2id::verify(¶ms, password, salt, empty).is_err()); + Argon2id::verify(¶ms, password, salt, too_short).expect_err("Argon2id must reject a short hash"); + Argon2id::verify(¶ms, password, salt, too_long).expect_err("Argon2id must reject a long hash"); + Argon2id::verify(¶ms, password, salt, empty).expect_err("Argon2id must reject an empty hash"); // And a length one byte off in either direction — the boundary cases. let off_minus_one: &[u8] = &hash[..31]; let off_plus_one: &[u8] = &[hash.as_ref(), &[0u8; 1]].concat(); - assert!(Argon2id::verify(¶ms, password, salt, off_minus_one).is_err()); - assert!(Argon2id::verify(¶ms, password, salt, off_plus_one).is_err()); + Argon2id::verify(¶ms, password, salt, off_minus_one).expect_err("Argon2id must reject a hash one byte too short"); + Argon2id::verify(¶ms, password, salt, off_plus_one).expect_err("Argon2id must reject a hash one byte too long"); } #[test] fn argon2id_verify_rejects_byte_flip_at_every_position() { - let params = Argon2Params::new(32, 2, 1).unwrap(); + let params = Argon2Params::new(32, 2, 1).expect("byte-flip test parameters must be valid"); let password = b"correct horse battery staple"; let salt = b"random-salt-1234"; let mut hash = [0u8; 32]; - Argon2id::derive(¶ms, password, salt, &mut hash).unwrap(); + Argon2id::derive(¶ms, password, salt, &mut hash).expect("byte-flip fixture must derive"); for pos in 0..hash.len() { let mut tampered = hash; tampered[pos] ^= 0x01; - assert!( - Argon2id::verify(¶ms, password, salt, &tampered).is_err(), - "verify must reject flip at byte {pos}" - ); + Argon2id::verify(¶ms, password, salt, &tampered) + .expect_err("Argon2id must reject a hash with any flipped byte"); } } diff --git a/tests/argon2_kernels.rs b/tests/argon2_kernels.rs index 99d8dd1b..92c291b1 100644 --- a/tests/argon2_kernels.rs +++ b/tests/argon2_kernels.rs @@ -13,15 +13,14 @@ //! full hash with a non-trivial cost matrix surfaces it. #![cfg(all(feature = "argon2", feature = "diag"))] -#![allow(clippy::unwrap_used)] use rscrypto::{ Argon2Params, auth::{argon2, argon2::Argon2Variant}, }; -fn params(m_kib: u32, t: u32, p: u32, _out_len: u32) -> Argon2Params { - Argon2Params::new(m_kib, t, p).unwrap() +fn params(m_kib: u32, t: u32, p: u32) -> Argon2Params { + Argon2Params::new(m_kib, t, p).expect("Argon2 kernel-test parameters must be valid") } const PASSWORD: &[u8] = b"correct horse battery staple"; @@ -30,12 +29,14 @@ const SALT: &[u8] = b"rscrypto-salt-16b!!!"; /// Every kernel must agree at a range of cost points. fn all_kernels_agree(variant: Argon2Variant) { for &(m, t, p) in &[(16u32, 1u32, 1u32), (32, 2, 1), (64, 1, 2), (32, 3, 1)] { - let params = params(m, t, p, 32); + let params = params(m, t, p); let mut expected = [0u8; 32]; - argon2::diag_hash_portable(¶ms, PASSWORD, SALT, variant, &mut expected).unwrap(); + argon2::diag_hash_portable(¶ms, PASSWORD, SALT, variant, &mut expected) + .expect("portable Argon2 kernel-test hash must succeed"); let mut active_out = [0u8; 32]; - argon2::diag_hash_active(¶ms, PASSWORD, SALT, variant, &mut active_out).unwrap(); + argon2::diag_hash_active(¶ms, PASSWORD, SALT, variant, &mut active_out) + .expect("active Argon2 kernel-test hash must succeed"); assert_eq!( active_out, expected, "active kernel diverged on m={m} t={t} p={p} variant={variant:?}" @@ -44,7 +45,8 @@ fn all_kernels_agree(variant: Argon2Variant) { #[cfg(target_arch = "aarch64")] { let mut neon_out = [0u8; 32]; - argon2::diag_hash_aarch64_neon(¶ms, PASSWORD, SALT, variant, &mut neon_out).unwrap(); + argon2::diag_hash_aarch64_neon(¶ms, PASSWORD, SALT, variant, &mut neon_out) + .expect("AArch64 NEON Argon2 kernel-test hash must succeed"); assert_eq!( neon_out, expected, "aarch64-neon diverged on m={m} t={t} p={p} variant={variant:?}" @@ -57,7 +59,8 @@ fn all_kernels_agree(variant: Argon2Variant) { if host.has(rscrypto::auth::argon2::required_caps(argon2::KernelId::X86Avx2)) { let mut out = [0u8; 32]; - argon2::diag_hash_x86_avx2(¶ms, PASSWORD, SALT, variant, &mut out).unwrap(); + argon2::diag_hash_x86_avx2(¶ms, PASSWORD, SALT, variant, &mut out) + .expect("x86 AVX2 Argon2 kernel-test hash must succeed"); assert_eq!( out, expected, "x86-avx2 diverged on m={m} t={t} p={p} variant={variant:?}" @@ -66,7 +69,8 @@ fn all_kernels_agree(variant: Argon2Variant) { if host.has(rscrypto::auth::argon2::required_caps(argon2::KernelId::X86Avx512)) { let mut out = [0u8; 32]; - argon2::diag_hash_x86_avx512(¶ms, PASSWORD, SALT, variant, &mut out).unwrap(); + argon2::diag_hash_x86_avx512(¶ms, PASSWORD, SALT, variant, &mut out) + .expect("x86 AVX-512 Argon2 kernel-test hash must succeed"); assert_eq!( out, expected, "x86-avx512 diverged on m={m} t={t} p={p} variant={variant:?}" @@ -77,7 +81,8 @@ fn all_kernels_agree(variant: Argon2Variant) { #[cfg(target_arch = "powerpc64")] if rscrypto::platform::caps().has(rscrypto::auth::argon2::required_caps(argon2::KernelId::PowerVsx)) { let mut out = [0u8; 32]; - argon2::diag_hash_power_vsx(¶ms, PASSWORD, SALT, variant, &mut out).unwrap(); + argon2::diag_hash_power_vsx(¶ms, PASSWORD, SALT, variant, &mut out) + .expect("POWER VSX Argon2 kernel-test hash must succeed"); assert_eq!( out, expected, "power-vsx diverged on m={m} t={t} p={p} variant={variant:?}" @@ -87,7 +92,8 @@ fn all_kernels_agree(variant: Argon2Variant) { #[cfg(target_arch = "s390x")] if rscrypto::platform::caps().has(rscrypto::auth::argon2::required_caps(argon2::KernelId::S390xVector)) { let mut out = [0u8; 32]; - argon2::diag_hash_s390x_vector(¶ms, PASSWORD, SALT, variant, &mut out).unwrap(); + argon2::diag_hash_s390x_vector(¶ms, PASSWORD, SALT, variant, &mut out) + .expect("s390x vector Argon2 kernel-test hash must succeed"); assert_eq!( out, expected, "s390x-vector diverged on m={m} t={t} p={p} variant={variant:?}" @@ -97,7 +103,8 @@ fn all_kernels_agree(variant: Argon2Variant) { #[cfg(target_arch = "riscv64")] if rscrypto::platform::caps().has(rscrypto::auth::argon2::required_caps(argon2::KernelId::Riscv64V)) { let mut out = [0u8; 32]; - argon2::diag_hash_riscv64_v(¶ms, PASSWORD, SALT, variant, &mut out).unwrap(); + argon2::diag_hash_riscv64_v(¶ms, PASSWORD, SALT, variant, &mut out) + .expect("RISC-V vector Argon2 kernel-test hash must succeed"); assert_eq!( out, expected, "riscv64-v diverged on m={m} t={t} p={p} variant={variant:?}" @@ -107,7 +114,8 @@ fn all_kernels_agree(variant: Argon2Variant) { #[cfg(target_arch = "wasm32")] if rscrypto::platform::caps().has(rscrypto::auth::argon2::required_caps(argon2::KernelId::WasmSimd128)) { let mut out = [0u8; 32]; - argon2::diag_hash_wasm_simd128(¶ms, PASSWORD, SALT, variant, &mut out).unwrap(); + argon2::diag_hash_wasm_simd128(¶ms, PASSWORD, SALT, variant, &mut out) + .expect("WASM SIMD128 Argon2 kernel-test hash must succeed"); assert_eq!( out, expected, "wasm-simd128 diverged on m={m} t={t} p={p} variant={variant:?}" @@ -136,9 +144,16 @@ fn argon2id_all_kernels_agree() { #[test] fn single_block_compress_matches_across_kernels() { use argon2::DIAG_BLOCK_WORDS; - let x: [u64; DIAG_BLOCK_WORDS] = core::array::from_fn(|i| (i as u64).wrapping_mul(0x0f0f_0f0f_0f0f_0f0f)); - let y: [u64; DIAG_BLOCK_WORDS] = - core::array::from_fn(|i| (i as u64).wrapping_mul(0xa5a5_a5a5_a5a5_a5a5).rotate_left(i as u32)); + let x: [u64; DIAG_BLOCK_WORDS] = core::array::from_fn(|index| { + u64::try_from(index) + .expect("Argon2 block-word index must fit in u64") + .wrapping_mul(0x0f0f_0f0f_0f0f_0f0f) + }); + let y: [u64; DIAG_BLOCK_WORDS] = core::array::from_fn(|index| { + let index_word = u64::try_from(index).expect("Argon2 block-word index must fit in u64"); + let rotation = u32::try_from(index).expect("Argon2 block-word index must fit in u32"); + index_word.wrapping_mul(0xa5a5_a5a5_a5a5_a5a5).rotate_left(rotation) + }); for xor_into in [false, true] { let mut expected = if xor_into { diff --git a/tests/argon2_miri.rs b/tests/argon2_miri.rs index 8460ee76..fd9111a6 100644 --- a/tests/argon2_miri.rs +++ b/tests/argon2_miri.rs @@ -62,8 +62,10 @@ fn argon2id_minimal_verify_no_ub() { let params = Argon2Params::new(8, 1, 1).expect("minimal params are valid"); let mut hash = [0u8; 4]; - Argon2id::derive(¶ms, b"correct", b"saltsalt", &mut hash).unwrap(); + Argon2id::derive(¶ms, b"correct", b"saltsalt", &mut hash) + .expect("Argon2id Miri verification fixture must derive"); - assert!(Argon2id::verify(¶ms, b"correct", b"saltsalt", &hash).is_ok()); - assert!(Argon2id::verify(¶ms, b"wrong", b"saltsalt", &hash).is_err()); + Argon2id::verify(¶ms, b"correct", b"saltsalt", &hash).expect("fresh Argon2id Miri hash must verify"); + Argon2id::verify(¶ms, b"wrong", b"saltsalt", &hash) + .expect_err("Argon2id Miri verification must reject the wrong password"); } diff --git a/tests/argon2_parallel.rs b/tests/argon2_parallel.rs index 99eb5f2a..b477567d 100644 --- a/tests/argon2_parallel.rs +++ b/tests/argon2_parallel.rs @@ -12,8 +12,8 @@ use rscrypto::{Argon2Params, Argon2d, Argon2i, Argon2id}; -fn rs_params(m_kib: u32, t: u32, p: u32, _out_len: u32) -> Argon2Params { - Argon2Params::new(m_kib, t, p).unwrap() +fn rs_params(m_kib: u32, t: u32, p: u32) -> Argon2Params { + Argon2Params::new(m_kib, t, p).expect("Argon2 parallel-test parameters must be valid") } fn oracle_hash( @@ -25,10 +25,13 @@ fn oracle_hash( p: u32, out_len: usize, ) -> Vec { - let params = argon2::Params::new(m_kib, t, p, Some(out_len)).unwrap(); + let params = argon2::Params::new(m_kib, t, p, Some(out_len)) + .expect("RustCrypto must accept valid Argon2 parallel-test parameters"); let ctx = argon2::Argon2::new(algo, argon2::Version::V0x13, params); let mut out = vec![0u8; out_len]; - ctx.hash_password_into(password, salt, &mut out).unwrap(); + ctx + .hash_password_into(password, salt, &mut out) + .expect("RustCrypto Argon2 parallel-test derivation must succeed"); out } @@ -46,9 +49,9 @@ fn argon2id_p8_matches_oracle() { let p = 8u32; let out_len = 32usize; - let params = rs_params(m, t, p, out_len as u32); + let params = rs_params(m, t, p); let mut actual = vec![0u8; out_len]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut actual).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut actual).expect("Argon2id p=8 derivation must succeed"); let expected = oracle_hash(argon2::Algorithm::Argon2id, PASSWORD, SALT, m, t, p, out_len); assert_eq!(actual, expected, "argon2id p=8 mismatch vs RustCrypto oracle"); @@ -61,9 +64,9 @@ fn argon2d_p8_matches_oracle() { let p = 8u32; let out_len = 32usize; - let params = rs_params(m, t, p, out_len as u32); + let params = rs_params(m, t, p); let mut actual = vec![0u8; out_len]; - Argon2d::derive(¶ms, PASSWORD, SALT, &mut actual).unwrap(); + Argon2d::derive(¶ms, PASSWORD, SALT, &mut actual).expect("Argon2d p=8 derivation must succeed"); let expected = oracle_hash(argon2::Algorithm::Argon2d, PASSWORD, SALT, m, t, p, out_len); assert_eq!(actual, expected, "argon2d p=8 mismatch vs RustCrypto oracle"); @@ -76,9 +79,9 @@ fn argon2i_p8_matches_oracle() { let p = 8u32; let out_len = 32usize; - let params = rs_params(m, t, p, out_len as u32); + let params = rs_params(m, t, p); let mut actual = vec![0u8; out_len]; - Argon2i::derive(¶ms, PASSWORD, SALT, &mut actual).unwrap(); + Argon2i::derive(¶ms, PASSWORD, SALT, &mut actual).expect("Argon2i p=8 derivation must succeed"); let expected = oracle_hash(argon2::Algorithm::Argon2i, PASSWORD, SALT, m, t, p, out_len); assert_eq!(actual, expected, "argon2i p=8 mismatch vs RustCrypto oracle"); @@ -91,9 +94,9 @@ fn argon2id_p16_matches_oracle() { let p = 16u32; let out_len = 32usize; - let params = rs_params(m, t, p, out_len as u32); + let params = rs_params(m, t, p); let mut actual = vec![0u8; out_len]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut actual).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut actual).expect("Argon2id p=16 derivation must succeed"); let expected = oracle_hash(argon2::Algorithm::Argon2id, PASSWORD, SALT, m, t, p, out_len); assert_eq!(actual, expected, "argon2id p=16 mismatch vs RustCrypto oracle"); @@ -105,28 +108,29 @@ fn argon2id_p16_matches_oracle() { /// produce the same tag. Any non-determinism would indicate a data race. #[test] fn argon2id_parallel_is_deterministic() { - let params = rs_params(1024, 2, 8, 32); + let params = rs_params(1024, 2, 8); let mut reference = [0u8; 32]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut reference).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut reference) + .expect("reference Argon2id parallel derivation must succeed"); for trial in 0..16 { let mut out = [0u8; 32]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut out).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut out).expect("repeated Argon2id derivation must succeed"); assert_eq!(out, reference, "non-deterministic output on trial {trial}"); } } #[test] fn argon2d_parallel_is_deterministic() { - let params = rs_params(1024, 2, 8, 32); + let params = rs_params(1024, 2, 8); let mut reference = [0u8; 32]; - Argon2d::derive(¶ms, PASSWORD, SALT, &mut reference).unwrap(); + Argon2d::derive(¶ms, PASSWORD, SALT, &mut reference).expect("reference Argon2d parallel derivation must succeed"); for trial in 0..16 { let mut out = [0u8; 32]; - Argon2d::derive(¶ms, PASSWORD, SALT, &mut out).unwrap(); + Argon2d::derive(¶ms, PASSWORD, SALT, &mut out).expect("repeated Argon2d derivation must succeed"); assert_eq!(out, reference, "non-deterministic output on trial {trial}"); } } @@ -142,9 +146,9 @@ fn argon2id_p1_fast_path_matches_oracle() { let p = 1u32; let out_len = 32usize; - let params = rs_params(m, t, p, out_len as u32); + let params = rs_params(m, t, p); let mut actual = vec![0u8; out_len]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut actual).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut actual).expect("Argon2id p=1 derivation must succeed"); let expected = oracle_hash(argon2::Algorithm::Argon2id, PASSWORD, SALT, m, t, p, out_len); assert_eq!(actual, expected, "argon2id p=1 fast-path mismatch vs oracle"); @@ -152,10 +156,11 @@ fn argon2id_p1_fast_path_matches_oracle() { #[test] fn argon2id_parallel_verify_round_trip() { - let params = rs_params(256, 2, 8, 32); + let params = rs_params(256, 2, 8); let mut hash = [0u8; 32]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut hash).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut hash).expect("Argon2id verification fixture must derive"); - assert!(Argon2id::verify(¶ms, PASSWORD, SALT, &hash).is_ok()); - assert!(Argon2id::verify(¶ms, b"wrong-password-zzzzzzzzzzzzzzzzzz", SALT, &hash).is_err()); + Argon2id::verify(¶ms, PASSWORD, SALT, &hash).expect("fresh Argon2id hash must verify"); + Argon2id::verify(¶ms, b"wrong-password-zzzzzzzzzzzzzzzzzz", SALT, &hash) + .expect_err("Argon2id must reject the wrong password"); } diff --git a/tests/argon2_vectors.rs b/tests/argon2_vectors.rs index e7fafc3f..ad88bfe2 100644 --- a/tests/argon2_vectors.rs +++ b/tests/argon2_vectors.rs @@ -33,7 +33,7 @@ fn rfc9106_appendix_a1_argon2d() { RFC_SALT, &mut out, ) - .unwrap(); + .expect("RFC 9106 Argon2d vector derivation must succeed"); assert_eq!(out, expected); } @@ -51,7 +51,7 @@ fn rfc9106_appendix_a2_argon2i() { RFC_SALT, &mut out, ) - .unwrap(); + .expect("RFC 9106 Argon2i vector derivation must succeed"); assert_eq!(out, expected); } @@ -69,7 +69,7 @@ fn rfc9106_appendix_a3_argon2id() { RFC_SALT, &mut out, ) - .unwrap(); + .expect("RFC 9106 Argon2id vector derivation must succeed"); assert_eq!(out, expected); } @@ -82,9 +82,12 @@ fn all_three_variants_produce_distinct_output() { let mut d = [0u8; 32]; let mut i = [0u8; 32]; let mut id = [0u8; 32]; - Argon2d::derive_with_context(¶ms, canonical_context(), RFC_PASSWORD, RFC_SALT, &mut d).unwrap(); - Argon2i::derive_with_context(¶ms, canonical_context(), RFC_PASSWORD, RFC_SALT, &mut i).unwrap(); - Argon2id::derive_with_context(¶ms, canonical_context(), RFC_PASSWORD, RFC_SALT, &mut id).unwrap(); + Argon2d::derive_with_context(¶ms, canonical_context(), RFC_PASSWORD, RFC_SALT, &mut d) + .expect("Argon2d distinct-output fixture must derive"); + Argon2i::derive_with_context(¶ms, canonical_context(), RFC_PASSWORD, RFC_SALT, &mut i) + .expect("Argon2i distinct-output fixture must derive"); + Argon2id::derive_with_context(¶ms, canonical_context(), RFC_PASSWORD, RFC_SALT, &mut id) + .expect("Argon2id distinct-output fixture must derive"); assert_ne!(d, i); assert_ne!(d, id); assert_ne!(i, id); diff --git a/tests/ascon_aead_oracle.rs b/tests/ascon_aead_oracle.rs index ad8454a5..800bd718 100644 --- a/tests/ascon_aead_oracle.rs +++ b/tests/ascon_aead_oracle.rs @@ -36,13 +36,15 @@ fn assert_matches_oracle(key_bytes: &[u8; 16], nonce_bytes: &[u8; 16], aad: &[u8 // Encrypt with rscrypto. let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto must seal valid Ascon-AEAD128 oracle input"); // Encrypt with oracle. let mut oracle_buf = plaintext.to_vec(); let oracle_tag = oracle .encrypt_inout_detached(&oracle_nonce, aad, oracle_buf.as_mut_slice().into()) - .unwrap(); + .expect("RustCrypto must seal valid Ascon-AEAD128 oracle input"); assert_eq!(ours, oracle_buf, "ciphertext mismatch (len={})", plaintext.len()); assert_eq!( @@ -53,7 +55,9 @@ fn assert_matches_oracle(key_bytes: &[u8; 16], nonce_bytes: &[u8; 16], aad: &[u8 ); // Decrypt with rscrypto. - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh Ascon-AEAD128 ciphertext must authenticate"); assert_eq!(ours, plaintext, "decrypt round-trip failed (len={})", plaintext.len()); } @@ -101,7 +105,7 @@ fn ascon_aead128_oracle_rate_boundary_sizes() { fn ascon_aead128_oracle_large_input() { let key = [0x77u8; 16]; let nonce = [0x88u8; 16]; - let plaintext: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let plaintext: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, b"large", &plaintext); } @@ -114,14 +118,15 @@ fn ascon_aead128_rejects_modified_tag() { let cipher = AsconAead128::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("Ascon-AEAD128 tag-forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &AsconAead128Tag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &AsconAead128Tag::from_bytes(tag)) + .expect_err("Ascon-AEAD128 must reject a modified tag"); } #[test] @@ -131,10 +136,14 @@ fn ascon_aead128_rejects_modified_ciphertext() { let cipher = AsconAead128::new(&key); let mut buffer = *b"tamper-detect"; - let tag = cipher.encrypt_in_place(&nonce, b"", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"", &mut buffer) + .expect("Ascon-AEAD128 ciphertext-tampering fixture must seal"); buffer[0] ^= 1; - assert!(cipher.decrypt_in_place(&nonce, b"", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"", &mut buffer, &tag) + .expect_err("Ascon-AEAD128 must reject modified ciphertext"); } #[test] @@ -144,7 +153,11 @@ fn ascon_aead128_rejects_wrong_aad() { let cipher = AsconAead128::new(&key); let mut buffer = *b"aad-mismatch"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buffer) + .expect("Ascon-AEAD128 AAD-mismatch fixture must seal"); - assert!(cipher.decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag) + .expect_err("Ascon-AEAD128 must reject incorrect associated data"); } diff --git a/tests/ascon_cxof_vectors.rs b/tests/ascon_cxof_vectors.rs index 09e837c7..701d2cdc 100644 --- a/tests/ascon_cxof_vectors.rs +++ b/tests/ascon_cxof_vectors.rs @@ -19,7 +19,8 @@ fn ascon_cxof128_matches_ascon_c_kat_vector() { "4F50159EF70BB3DAD8807E034EAEBD44C4FA2CBBC8CF1F05511AB66CDCC529905CA12083FC186AD899B270B1473DC5F7EC88D1052082DCDFE69FB75D269E7B74", ); - let actual = squeeze_all(AsconCxof128::xof(&customization, &msg).unwrap(), expected.len()); + let reader = AsconCxof128::xof(&customization, &msg).expect("empty Ascon-CXOF128 customization must be valid"); + let actual = squeeze_all(reader, expected.len()); assert_eq!(actual, expected); } @@ -28,7 +29,7 @@ fn ascon_cxof128_reset_restores_customized_state() { let customization = b"ctx=v1"; let data = b"abc"; - let mut hasher = AsconCxof128::new(customization).unwrap(); + let mut hasher = AsconCxof128::new(customization).expect("short Ascon-CXOF128 customization must be valid"); hasher.update(data); let expected = squeeze_all(hasher.finalize_xof(), 64); @@ -40,6 +41,6 @@ fn ascon_cxof128_reset_restores_customized_state() { #[test] fn ascon_cxof128_rejects_long_customization() { - let err = AsconCxof128::new(&[0u8; 257]).unwrap_err(); + let err = AsconCxof128::new(&[0u8; 257]).expect_err("257-byte Ascon-CXOF128 customization must be rejected"); assert_eq!(err.to_string(), "Ascon-CXOF128 customization exceeds 256 bytes"); } diff --git a/tests/ascon_differential.rs b/tests/ascon_differential.rs index 94fd2407..5040c374 100644 --- a/tests/ascon_differential.rs +++ b/tests/ascon_differential.rs @@ -10,8 +10,8 @@ fn ascon_hash256_streaming(data: &[u8]) -> [u8; 32] { let mut hasher = AsconHash256::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = usize::from(data[i]).strict_rem(97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); hasher.update(&data[i..end]); i = end; } @@ -57,8 +57,8 @@ proptest! { split_data in any::(), split_out in any::(), ) { - let split_data = split_data % (data.len() + 1); - let split_out = split_out % (out_len + 1); + let split_data = split_data.strict_rem(data.len().strict_add(1)); + let split_out = split_out.strict_rem(out_len.strict_add(1)); let mut expected = vec![0u8; out_len]; { diff --git a/tests/ascon_final_kats.rs b/tests/ascon_final_kats.rs index 857a1ad3..11fd8d6c 100644 --- a/tests/ascon_final_kats.rs +++ b/tests/ascon_final_kats.rs @@ -3,8 +3,11 @@ mod common; #[cfg(feature = "ascon-aead")] -use common::decode_hex_array; +#[path = "common/array.rs"] +mod hex_array; use common::decode_hex_vec; +#[cfg(feature = "ascon-aead")] +use hex_array::decode_hex_array; #[cfg(feature = "ascon-aead")] const AEAD_VECTORS: &str = include_str!("../testdata/ascon/asconaead128.txt"); @@ -20,11 +23,13 @@ fn field<'a>(case: &'a str, name: &str) -> &'a str { .lines() .find_map(|line| line.split_once(" = ").filter(|(key, _)| *key == name)) .map(|(_, value)| value) - .unwrap_or_else(|| panic!("missing Ascon KAT field `{name}`")) + .expect("Ascon KAT field must be present") } fn case_number(case: &str) -> usize { - field(case, "Count").parse().expect("invalid Ascon KAT case number") + field(case, "Count") + .parse() + .expect("Ascon KAT case number must be valid") } #[cfg(feature = "ascon-aead")] @@ -35,7 +40,7 @@ fn ascon_aead128_matches_final_reference_kats() { aead::{Nonce128, expert::AeadWithNonce as _}, }; - let mut rows = 0; + let mut rows = 0usize; for case in cases(AEAD_VECTORS) { let count = case_number(case); let key = AsconAead128Key::from_bytes(decode_hex_array(field(case, "Key"))); @@ -58,7 +63,7 @@ fn ascon_aead128_matches_final_reference_kats() { let mut ciphertext = plaintext.clone(); let tag = cipher .encrypt_in_place(&nonce, &aad, &mut ciphertext) - .unwrap_or_else(|err| panic!("Ascon-AEAD128 KAT {count} encryption failed: {err}")); + .expect("Ascon-AEAD128 KAT encryption must succeed"); assert_eq!( ciphertext, expected_ciphertext, "Ascon-AEAD128 KAT {count} ciphertext mismatch" @@ -72,10 +77,10 @@ fn ascon_aead128_matches_final_reference_kats() { let mut decrypted = expected_ciphertext.to_vec(); cipher .decrypt_in_place(&nonce, &aad, &mut decrypted, &expected_tag) - .unwrap_or_else(|err| panic!("Ascon-AEAD128 KAT {count} decryption failed: {err}")); + .expect("Ascon-AEAD128 KAT decryption must succeed"); assert_eq!(decrypted, plaintext, "Ascon-AEAD128 KAT {count} plaintext mismatch"); - rows += 1; + rows = rows.strict_add(1); } assert_eq!(rows, 1089, "incomplete Ascon-AEAD128 KAT corpus"); } @@ -85,7 +90,7 @@ fn ascon_aead128_matches_final_reference_kats() { fn ascon_cxof128_matches_final_reference_kats() { use rscrypto::{AsconCxof128, traits::Xof as _}; - let mut rows = 0; + let mut rows = 0usize; for case in cases(CXOF_VECTORS) { let count = case_number(case); let message = decode_hex_vec(field(case, "Msg")); @@ -94,11 +99,11 @@ fn ascon_cxof128_matches_final_reference_kats() { let mut actual = vec![0u8; expected.len()]; AsconCxof128::xof(&customization, &message) - .unwrap_or_else(|err| panic!("Ascon-CXOF128 KAT {count} setup failed: {err}")) + .expect("Ascon-CXOF128 KAT setup must succeed") .squeeze(&mut actual); assert_eq!(actual, expected, "Ascon-CXOF128 KAT {count} output mismatch"); - rows += 1; + rows = rows.strict_add(1); } assert_eq!(rows, 1089, "incomplete Ascon-CXOF128 KAT corpus"); } diff --git a/tests/ascon_hash_oracle.rs b/tests/ascon_hash_oracle.rs index 04dc0333..7c88cc91 100644 --- a/tests/ascon_hash_oracle.rs +++ b/tests/ascon_hash_oracle.rs @@ -56,7 +56,7 @@ fn ascon_hash256_oracle_rate_boundaries() { #[test] fn ascon_hash256_oracle_large() { - let input: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let input: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_hash_matches_oracle(&input); } @@ -133,7 +133,7 @@ fn ascon_xof128_oracle_large_squeeze() { #[test] fn ascon_xof128_oracle_large_input() { - let input: Vec = (0..4096).map(|i| (i & 0xFF) as u8).collect(); + let input: Vec = (0usize..4096).map(|i| i.to_le_bytes()[0]).collect(); assert_xof_matches_oracle(&input, 64); } diff --git a/tests/ascon_official_vectors.rs b/tests/ascon_official_vectors.rs index 1b6186cd..d4a0757d 100644 --- a/tests/ascon_official_vectors.rs +++ b/tests/ascon_official_vectors.rs @@ -6,17 +6,16 @@ use rscrypto::{ hashes::crypto::{AsconHash256, AsconXof}, traits::{Digest as _, Xof as _}, }; -use support::blobby_compat::Blob2Iterator; +use support::blobby_compat::BlobIterator; #[test] fn ascon_hash256_official_vectors() { let data = include_bytes!("../testdata/ascon/asconhash.blb"); - for (i, row) in Blob2Iterator::new(data) + for (i, row) in BlobIterator::<2>::new(data) .expect("ascon hash vector corpus must parse") .enumerate() { - let [input, output] = - row.unwrap_or_else(|err| panic!("ascon-hash256 vector row decode failed at case {i}: {err:?}")); + let [input, output] = row.expect("Ascon-Hash256 vector row must decode"); let actual = AsconHash256::digest(input); assert_eq!( &actual[..], @@ -30,12 +29,11 @@ fn ascon_hash256_official_vectors() { #[test] fn ascon_xof_official_vectors() { let data = include_bytes!("../testdata/ascon/asconxof.blb"); - for (i, row) in Blob2Iterator::new(data) + for (i, row) in BlobIterator::<2>::new(data) .expect("ascon xof vector corpus must parse") .enumerate() { - let [input, output] = - row.unwrap_or_else(|err| panic!("ascon-xof128 vector row decode failed at case {i}: {err:?}")); + let [input, output] = row.expect("Ascon-XOF128 vector row must decode"); let mut actual = vec![0u8; output.len()]; AsconXof::xof(input).squeeze(&mut actual); assert_eq!( diff --git a/tests/blake2_differential.rs b/tests/blake2_differential.rs index 547a02d3..b32f32cb 100644 --- a/tests/blake2_differential.rs +++ b/tests/blake2_differential.rs @@ -1,11 +1,14 @@ #![cfg(feature = "hashes")] use blake2::{ - Blake2b256 as OracleBlake2b256, Blake2b512 as OracleBlake2b512, Blake2bMac, Blake2s128 as OracleBlake2s128, - Blake2s256 as OracleBlake2s256, Blake2sMac, Digest as _, + Blake2b as OracleBlake2b, Blake2b512 as OracleBlake2b512, Blake2bMac, Blake2bVarCore, Blake2s as OracleBlake2s, + Blake2s256 as OracleBlake2s256, Blake2sMac, Blake2sVarCore, + digest::{ + Digest as _, Mac as _, Output, + consts::{U16, U32, U64}, + core_api::{Buffer, UpdateCore, VariableOutputCore}, + }, }; -use digest::typenum::{U16, U32, U64}; -use hmac::{Mac as _, digest::KeyInit}; use proptest::{prelude::*, test_runner::Config as ProptestConfig}; use rscrypto::{ Blake2b256, Blake2b512, Blake2bKey, Blake2bParams, Blake2s128, Blake2s256, Blake2sKey, Blake2sParams, Digest, @@ -15,18 +18,47 @@ type OracleBlake2bMac256 = Blake2bMac; type OracleBlake2bMac512 = Blake2bMac; type OracleBlake2sMac128 = Blake2sMac; type OracleBlake2sMac256 = Blake2sMac; +type OracleBlake2b256 = OracleBlake2b; +type OracleBlake2s128 = OracleBlake2s; fn split_at_ratio(data: &[u8], ratio: u8) -> (&[u8], &[u8]) { let idx = if data.is_empty() { 0 } else { - data.len().strict_mul(ratio as usize) / 255 + data.len().strict_mul(usize::from(ratio)) / 255 }; data.split_at(idx.min(data.len())) } fn patterned_input(seed: u8, len: usize) -> Vec { - (0..len).map(|i| seed.wrapping_add((i % 251) as u8)).collect() + (0..len) + .map(|i| { + let offset = u8::try_from(i % 251).expect("remainder modulo 251 must fit in one byte"); + seed.wrapping_add(offset) + }) + .collect() +} + +fn oracle_blake2b_unkeyed(data: &[u8], salt: &[u8], personal: &[u8]) -> [u8; N] { + let mut core = Blake2bVarCore::new_with_params(salt, personal, 0, N); + let mut buffer = Buffer::::default(); + buffer.digest_blocks(data, |blocks| core.update_blocks(blocks)); + let mut full = Output::::default(); + core.finalize_variable_core(&mut buffer, &mut full); + let mut out = [0u8; N]; + out.copy_from_slice(&full[..N]); + out +} + +fn oracle_blake2s_unkeyed(data: &[u8], salt: &[u8], personal: &[u8]) -> [u8; N] { + let mut core = Blake2sVarCore::new_with_params(salt, personal, 0, N); + let mut buffer = Buffer::::default(); + buffer.digest_blocks(data, |blocks| core.update_blocks(blocks)); + let mut full = Output::::default(); + core.finalize_variable_core(&mut buffer, &mut full); + let mut out = [0u8; N]; + out.copy_from_slice(&full[..N]); + out } proptest! { @@ -41,7 +73,7 @@ proptest! { ) { let (left, right) = split_at_ratio(&data, split); let key = &patterned_input(0x42, key_len); - let typed_key = Blake2bKey::new(key).unwrap(); + let typed_key = Blake2bKey::new(key).expect("generated BLAKE2b key length must be valid"); let tail = patterned_input(0xA5, tail_len); let expected_256 = OracleBlake2b256::digest(&data); @@ -59,7 +91,8 @@ proptest! { streaming_512.update(right); prop_assert_eq!(&streaming_512.finalize()[..], expected_512.as_slice()); - let mut oracle_keyed_256 = OracleBlake2bMac256::new_from_slice(key).unwrap(); + let mut oracle_keyed_256 = OracleBlake2bMac256::new_from_slice(key) + .expect("RustCrypto must accept the generated BLAKE2b-256 key"); oracle_keyed_256.update(&data); let expected_keyed_256 = oracle_keyed_256.finalize().into_bytes(); prop_assert_eq!( @@ -67,7 +100,8 @@ proptest! { &expected_keyed_256[..] ); - let mut oracle_keyed_512 = OracleBlake2bMac512::new_from_slice(key).unwrap(); + let mut oracle_keyed_512 = OracleBlake2bMac512::new_from_slice(key) + .expect("RustCrypto must accept the generated BLAKE2b-512 key"); oracle_keyed_512.update(&data); let expected_keyed_512 = oracle_keyed_512.finalize().into_bytes(); prop_assert_eq!( @@ -77,14 +111,14 @@ proptest! { let mut reset_256 = Blake2b256::new(); reset_256.update(&data); - let _ = reset_256.finalize(); + let _first_digest = reset_256.finalize(); reset_256.reset(); reset_256.update(&tail); prop_assert_eq!(reset_256.finalize(), Blake2b256::digest(&tail)); let mut reset_512 = Blake2b512::new(); reset_512.update(&data); - let _ = reset_512.finalize(); + let _first_digest = reset_512.finalize(); reset_512.reset(); reset_512.update(&tail); prop_assert_eq!(reset_512.finalize(), Blake2b512::digest(&tail)); @@ -99,7 +133,7 @@ proptest! { ) { let (left, right) = split_at_ratio(&data, split); let key = &patterned_input(0x24, key_len); - let typed_key = Blake2sKey::new(key).unwrap(); + let typed_key = Blake2sKey::new(key).expect("generated BLAKE2s key length must be valid"); let tail = patterned_input(0x5A, tail_len); let expected_128 = OracleBlake2s128::digest(&data); @@ -117,7 +151,8 @@ proptest! { streaming_256.update(right); prop_assert_eq!(&streaming_256.finalize()[..], expected_256.as_slice()); - let mut oracle_keyed_128 = OracleBlake2sMac128::new_from_slice(key).unwrap(); + let mut oracle_keyed_128 = OracleBlake2sMac128::new_from_slice(key) + .expect("RustCrypto must accept the generated BLAKE2s-128 key"); oracle_keyed_128.update(&data); let expected_keyed_128 = oracle_keyed_128.finalize().into_bytes(); prop_assert_eq!( @@ -125,7 +160,8 @@ proptest! { &expected_keyed_128[..] ); - let mut oracle_keyed_256 = OracleBlake2sMac256::new_from_slice(key).unwrap(); + let mut oracle_keyed_256 = OracleBlake2sMac256::new_from_slice(key) + .expect("RustCrypto must accept the generated BLAKE2s-256 key"); oracle_keyed_256.update(&data); let expected_keyed_256 = oracle_keyed_256.finalize().into_bytes(); prop_assert_eq!( @@ -135,14 +171,14 @@ proptest! { let mut reset_128 = Blake2s128::new(); reset_128.update(&data); - let _ = reset_128.finalize(); + let _first_digest = reset_128.finalize(); reset_128.reset(); reset_128.update(&tail); prop_assert_eq!(reset_128.finalize(), Blake2s128::digest(&tail)); let mut reset_256 = Blake2s256::new(); reset_256.update(&data); - let _ = reset_256.finalize(); + let _first_digest = reset_256.finalize(); reset_256.reset(); reset_256.update(&tail); prop_assert_eq!(reset_256.finalize(), Blake2s256::digest(&tail)); @@ -163,17 +199,18 @@ proptest! { let mut personal_field = [0u8; 16]; personal_field[..personal.len()].copy_from_slice(&personal); - let key_opt: Option<&[u8]> = if key.is_empty() { None } else { Some(&key) }; - - // Oracle: Blake2bMac::new_with_salt_and_personal(key, salt, personal). - // When key=None the MAC reduces to a plain keyless hash with params. - let mut oracle_256 = OracleBlake2bMac256::new_with_salt_and_personal(key_opt, &salt, &personal).unwrap(); - oracle_256.update(&data); - let expected_256 = oracle_256.finalize().into_bytes(); + let expected_256: [u8; 32] = if key.is_empty() { + oracle_blake2b_unkeyed(&data, &salt, &personal) + } else { + let mut oracle = OracleBlake2bMac256::new_with_salt_and_personal(&key, &salt, &personal) + .expect("RustCrypto must accept generated BLAKE2b-256 parameters"); + oracle.update(&data); + oracle.finalize().into_bytes().into() + }; let mut params = Blake2bParams::new().salt(salt_field).personal(personal_field); if !key.is_empty() { - params = params.key(Blake2bKey::new(&key).unwrap()); + params = params.key(Blake2bKey::new(&key).expect("generated BLAKE2b parameter key must be valid")); } let ours_oneshot_256 = params.hash_256(&data); prop_assert_eq!(&ours_oneshot_256[..], &expected_256[..]); @@ -183,9 +220,14 @@ proptest! { ours_stream_256.update(&data); prop_assert_eq!(&ours_stream_256.finalize()[..], &expected_256[..]); - let mut oracle_512 = OracleBlake2bMac512::new_with_salt_and_personal(key_opt, &salt, &personal).unwrap(); - oracle_512.update(&data); - let expected_512 = oracle_512.finalize().into_bytes(); + let expected_512: [u8; 64] = if key.is_empty() { + oracle_blake2b_unkeyed(&data, &salt, &personal) + } else { + let mut oracle = OracleBlake2bMac512::new_with_salt_and_personal(&key, &salt, &personal) + .expect("RustCrypto must accept generated BLAKE2b-512 parameters"); + oracle.update(&data); + oracle.finalize().into_bytes().into() + }; let ours_oneshot_512 = params.hash_512(&data); prop_assert_eq!(&ours_oneshot_512[..], &expected_512[..]); @@ -206,15 +248,18 @@ proptest! { let mut personal_field = [0u8; 8]; personal_field[..personal.len()].copy_from_slice(&personal); - let key_opt: Option<&[u8]> = if key.is_empty() { None } else { Some(&key) }; - - let mut oracle_256 = OracleBlake2sMac256::new_with_salt_and_personal(key_opt, &salt, &personal).unwrap(); - oracle_256.update(&data); - let expected_256 = oracle_256.finalize().into_bytes(); + let expected_256: [u8; 32] = if key.is_empty() { + oracle_blake2s_unkeyed(&data, &salt, &personal) + } else { + let mut oracle = OracleBlake2sMac256::new_with_salt_and_personal(&key, &salt, &personal) + .expect("RustCrypto must accept generated BLAKE2s-256 parameters"); + oracle.update(&data); + oracle.finalize().into_bytes().into() + }; let mut params = Blake2sParams::new().salt(salt_field).personal(personal_field); if !key.is_empty() { - params = params.key(Blake2sKey::new(&key).unwrap()); + params = params.key(Blake2sKey::new(&key).expect("generated BLAKE2s parameter key must be valid")); } let ours_oneshot_256 = params.hash_256(&data); prop_assert_eq!(&ours_oneshot_256[..], &expected_256[..]); @@ -223,9 +268,14 @@ proptest! { ours_stream_256.update(&data); prop_assert_eq!(&ours_stream_256.finalize()[..], &expected_256[..]); - let mut oracle_128 = OracleBlake2sMac128::new_with_salt_and_personal(key_opt, &salt, &personal).unwrap(); - oracle_128.update(&data); - let expected_128 = oracle_128.finalize().into_bytes(); + let expected_128: [u8; 16] = if key.is_empty() { + oracle_blake2s_unkeyed(&data, &salt, &personal) + } else { + let mut oracle = OracleBlake2sMac128::new_with_salt_and_personal(&key, &salt, &personal) + .expect("RustCrypto must accept generated BLAKE2s-128 parameters"); + oracle.update(&data); + oracle.finalize().into_bytes().into() + }; let ours_oneshot_128 = params.hash_128(&data); prop_assert_eq!(&ours_oneshot_128[..], &expected_128[..]); diff --git a/tests/blake2_official_vectors.rs b/tests/blake2_official_vectors.rs index 8dcf140f..025f211b 100644 --- a/tests/blake2_official_vectors.rs +++ b/tests/blake2_official_vectors.rs @@ -3,7 +3,7 @@ mod support; use rscrypto::{Blake2b512, Blake2bKey, Blake2s256, Blake2sKey, Digest}; -use support::blobby_compat::Blob3Iterator; +use support::blobby_compat::BlobIterator; fn run_blake2_vectors( data: &'static [u8], @@ -11,12 +11,11 @@ fn run_blake2_vectors( mut one_shot: impl FnMut(&[u8], &[u8]) -> [u8; OUT], mut streaming: impl FnMut(&[u8], &[u8]) -> [u8; OUT], ) { - for (index, row) in Blob3Iterator::new(data) + for (index, row) in BlobIterator::<3>::new(data) .expect("blake2 vector corpus must parse") .enumerate() { - let [input, key, output] = - row.unwrap_or_else(|err| panic!("{name} vector row decode failed at case {index}: {err:?}")); + let [input, key, output] = row.expect("BLAKE2 vector row must decode"); let actual = one_shot(input, key); assert_eq!( @@ -48,14 +47,16 @@ fn blake2s_official_vectors() { if key.is_empty() { Blake2s256::digest(input) } else { - Blake2s256::keyed_digest(Blake2sKey::new(key).unwrap(), input) + let key = Blake2sKey::new(key).expect("official BLAKE2s key length must be valid"); + Blake2s256::keyed_digest(key, input) } }, |input, key| { let mut hasher = if key.is_empty() { Blake2s256::new() } else { - Blake2s256::new_keyed(Blake2sKey::new(key).unwrap()) + let key = Blake2sKey::new(key).expect("official BLAKE2s key length must be valid"); + Blake2s256::new_keyed(key) }; for chunk in input.chunks(11) { hasher.update(chunk); @@ -75,14 +76,16 @@ fn blake2b_official_vectors() { if key.is_empty() { Blake2b512::digest(input) } else { - Blake2b512::keyed_digest(Blake2bKey::new(key).unwrap(), input) + let key = Blake2bKey::new(key).expect("official BLAKE2b key length must be valid"); + Blake2b512::keyed_digest(key, input) } }, |input, key| { let mut hasher = if key.is_empty() { Blake2b512::new() } else { - Blake2b512::new_keyed(Blake2bKey::new(key).unwrap()) + let key = Blake2bKey::new(key).expect("official BLAKE2b key length must be valid"); + Blake2b512::new_keyed(key) }; for chunk in input.chunks(17) { hasher.update(chunk); diff --git a/tests/blake3_differential.rs b/tests/blake3_differential.rs index 6d834803..c9a3ff4e 100644 --- a/tests/blake3_differential.rs +++ b/tests/blake3_differential.rs @@ -37,7 +37,9 @@ fn blake3_ref_derive_xof(context: &str, data: &[u8], out: &mut [u8]) { } fn patterned_bytes(len: usize) -> Vec { - (0..len).map(|i| (i % 251) as u8).collect() + (0..len) + .map(|i| u8::try_from(i % 251).expect("remainder modulo 251 must fit in one byte")) + .collect() } proptest! { @@ -53,8 +55,8 @@ proptest! { let mut h = Blake3::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 251) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 251).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } diff --git a/tests/blake3_official_vectors.rs b/tests/blake3_official_vectors.rs index 18ee7228..b80b05bd 100644 --- a/tests/blake3_official_vectors.rs +++ b/tests/blake3_official_vectors.rs @@ -3,7 +3,7 @@ mod support; use rscrypto::{Digest, hashes::crypto::Blake3, traits::Xof as _}; -use support::blobby_compat::Blob6Iterator; +use support::blobby_compat::BlobIterator; fn update_input_pattern(hasher: &mut Blake3, len: usize) { let mut remaining = len; @@ -12,11 +12,12 @@ fn update_input_pattern(hasher: &mut Blake3, len: usize) { while remaining != 0 { let take = core::cmp::min(remaining, buf.len()); for (i, b) in buf[..take].iter_mut().enumerate() { - *b = ((offset + i) % 251) as u8; + let value = offset.strict_add(i).strict_rem(251); + *b = u8::try_from(value).expect("BLAKE3 fixture byte must fit in u8"); } hasher.update(&buf[..take]); - offset += take; - remaining -= take; + offset = offset.strict_add(take); + remaining = remaining.strict_sub(take); } } @@ -28,7 +29,7 @@ fn decode_u64_le(bytes: &[u8]) -> u64 { #[test] fn blake3_official_test_vectors() { let blb = include_bytes!("../testdata/blake3/test_vectors.blb"); - for (i, row) in Blob6Iterator::new(blb) + for (i, row) in BlobIterator::<6>::new(blb) .expect("blake3 vector corpus must parse") .enumerate() { @@ -39,14 +40,15 @@ fn blake3_official_test_vectors() { hash_xof, keyed_hash_xof, derive_key_xof, - ] = row.unwrap_or_else(|err| panic!("blake3 vector row decode failed at case {i}: {err:?}")); + ] = row.expect("BLAKE3 vector row must decode"); assert_eq!(key_bytes.len(), 32, "blake3 key length mismatch at case {i}"); let mut key = [0u8; 32]; key.copy_from_slice(key_bytes); let context = core::str::from_utf8(context_bytes).expect("blake3 context_string is valid UTF-8"); - let input_len = decode_u64_le(input_len_bytes) as usize; + let input_len = + usize::try_from(decode_u64_le(input_len_bytes)).expect("BLAKE3 vector input length must fit in usize"); // Hash mode { diff --git a/tests/chacha20poly1305.rs b/tests/chacha20poly1305.rs index 834d24e9..cedec351 100644 --- a/tests/chacha20poly1305.rs +++ b/tests/chacha20poly1305.rs @@ -15,9 +15,10 @@ use common::decode_hex_vec as decode_hex; fn pattern_bytes(len: usize, seed: u8) -> Vec { let mut out = vec![0u8; len]; for (index, byte) in out.iter_mut().enumerate() { + let index = index.to_le_bytes()[0]; *byte = seed - .wrapping_add((index as u8).wrapping_mul(17)) - .wrapping_add((index as u8).rotate_left(1)); + .wrapping_add(index.wrapping_mul(17)) + .wrapping_add(index.rotate_left(1)); } out } @@ -35,18 +36,22 @@ fn chacha20poly1305_matches_rfc_8439_vector() { ); let expected_tag = decode_hex("1ae10b594f09e26a7e902ecbd0600691"); - let key = ChaCha20Poly1305Key::from_bytes(key.try_into().unwrap()); - let nonce = Nonce96::from_bytes(nonce.try_into().unwrap()); + let key = ChaCha20Poly1305Key::from_bytes(key.try_into().expect("RFC 8439 key must be 32 bytes")); + let nonce = Nonce96::from_bytes(nonce.try_into().expect("RFC 8439 nonce must be 12 bytes")); let cipher = ChaCha20Poly1305::new(&key); let mut sealed = vec![0u8; plaintext.len() + ChaCha20Poly1305::TAG_SIZE]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut sealed).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut sealed) + .expect("RFC 8439 seal buffer must fit plaintext and tag"); assert_eq!(&sealed[..plaintext.len()], expected_ciphertext.as_slice()); assert_eq!(&sealed[plaintext.len()..], expected_tag.as_slice()); let mut opened = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &aad, &sealed, &mut opened).unwrap(); + cipher + .decrypt(&nonce, &aad, &sealed, &mut opened) + .expect("RFC 8439 ciphertext and tag must authenticate"); assert_eq!(opened, plaintext); } @@ -65,18 +70,22 @@ fn chacha20poly1305_matches_rustcrypto_oracle() { let oracle_nonce = Array(nonce_bytes); let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto ChaCha20-Poly1305 oracle input must seal"); let mut oracle_buffer = plaintext.to_vec(); let oracle_tag = oracle .encrypt_inout_detached(&oracle_nonce, aad, oracle_buffer.as_mut_slice().into()) - .unwrap(); + .expect("RustCrypto ChaCha20-Poly1305 oracle input must seal"); assert_eq!(ours, oracle_buffer); assert_eq!(tag.as_bytes(), oracle_tag.as_slice()); let typed_tag = ChaCha20Poly1305Tag::from_bytes(tag.to_bytes()); - cipher.decrypt_in_place(&nonce, aad, &mut ours, &typed_tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &typed_tag) + .expect("fresh rscrypto ChaCha20-Poly1305 ciphertext must authenticate"); assert_eq!(ours, plaintext); } @@ -87,14 +96,15 @@ fn chacha20poly1305_rejects_modified_tag() { let cipher = ChaCha20Poly1305::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("ChaCha20-Poly1305 forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &ChaCha20Poly1305Tag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &ChaCha20Poly1305Tag::from_bytes(tag)) + .expect_err("modified ChaCha20-Poly1305 tag must fail authentication"); } #[cfg(feature = "diag")] @@ -110,9 +120,12 @@ fn chacha20poly1305_diag_owned_decrypt_large_inputs_match_normal_path() { let plaintext = pattern_bytes(plaintext_len, 0x51); let aad = pattern_bytes(257, 0xa7); let mut ciphertext = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut ciphertext).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut ciphertext) + .expect("diagnostic ChaCha20-Poly1305 fixture must seal"); - rscrypto::aead::diag_chacha20poly1305_decrypt_in_place_owned(&cipher, &nonce, &aad, &mut ciphertext, &tag).unwrap(); + rscrypto::aead::diag_chacha20poly1305_decrypt_in_place_owned(&cipher, &nonce, &aad, &mut ciphertext, &tag) + .expect("owned diagnostic ChaCha20-Poly1305 path must open its fixture"); assert_eq!( ciphertext, plaintext, @@ -130,10 +143,10 @@ fn chacha20poly1305_diag_owned_decrypt_zeroes_large_buffer_on_bad_tag() { let plaintext = pattern_bytes(4097, 0x6d); let aad = pattern_bytes(33, 0x95); - let mut ciphertext = plaintext.clone(); + let mut ciphertext = plaintext; let mut tag = cipher .encrypt_in_place(&nonce, &aad, &mut ciphertext) - .unwrap() + .expect("diagnostic ChaCha20-Poly1305 bad-tag fixture must seal") .to_bytes(); tag[7] ^= 0x80; @@ -145,7 +158,7 @@ fn chacha20poly1305_diag_owned_decrypt_zeroes_large_buffer_on_bad_tag() { &ChaCha20Poly1305Tag::from_bytes(tag), ); - assert!(result.is_err()); + result.expect_err("owned diagnostic ChaCha20-Poly1305 path must reject a modified tag"); assert!( ciphertext.iter().all(|&byte| byte == 0), "owned diagnostic decrypt must zero caller buffer on verification failure" @@ -189,10 +202,11 @@ fn chacha20poly1305_diag_x86_asm_encrypt_matches_owned_path() { let mut expected = plaintext.clone(); let expected_tag = - rscrypto::aead::diag_chacha20poly1305_encrypt_in_place_owned(&cipher, &nonce, &aad, &mut expected).unwrap(); + rscrypto::aead::diag_chacha20poly1305_encrypt_in_place_owned(&cipher, &nonce, &aad, &mut expected) + .expect("owned diagnostic ChaCha20-Poly1305 path must seal"); let actual_tag = actual_tag .expect("x86 asm path must apply when AVX2+BMI2 are present") - .unwrap(); + .expect("x86 assembly ChaCha20-Poly1305 path must seal"); assert_eq!( actual, expected, @@ -208,10 +222,10 @@ fn chacha20poly1305_diag_x86_asm_encrypt_matches_owned_path() { #[test] fn chacha20poly1305_rejects_wrong_tag_length() { - assert!(ChaCha20Poly1305::tag_from_slice(&[0u8; 0]).is_err()); - assert!(ChaCha20Poly1305::tag_from_slice(&[0u8; 15]).is_err()); - assert!(ChaCha20Poly1305::tag_from_slice(&[0u8; 17]).is_err()); - assert!(ChaCha20Poly1305::tag_from_slice(&[0u8; 16]).is_ok()); + ChaCha20Poly1305::tag_from_slice(&[0u8; 0]).expect_err("empty ChaCha20-Poly1305 tag must be rejected"); + ChaCha20Poly1305::tag_from_slice(&[0u8; 15]).expect_err("short ChaCha20-Poly1305 tag must be rejected"); + ChaCha20Poly1305::tag_from_slice(&[0u8; 17]).expect_err("long ChaCha20-Poly1305 tag must be rejected"); + let _tag = ChaCha20Poly1305::tag_from_slice(&[0u8; 16]).expect("16-byte ChaCha20-Poly1305 tag must be accepted"); } #[test] @@ -237,7 +251,9 @@ fn chacha20poly1305_boundary_and_large_inputs_match_oracle() { let aad = pattern_bytes(aad_len, 0x9b); let mut combined = vec![0u8; plaintext_len + ChaCha20Poly1305::TAG_SIZE]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut combined).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut combined) + .expect("rscrypto combined ChaCha20-Poly1305 oracle input must seal"); let oracle_combined = oracle .encrypt( @@ -247,14 +263,16 @@ fn chacha20poly1305_boundary_and_large_inputs_match_oracle() { aad: &aad, }, ) - .unwrap(); + .expect("RustCrypto combined ChaCha20-Poly1305 oracle input must seal"); assert_eq!( combined, oracle_combined, "combined ciphertext mismatch pt_len={plaintext_len} aad_len={aad_len}" ); let mut opened = vec![0u8; plaintext_len]; - cipher.decrypt(&nonce, &aad, &oracle_combined, &mut opened).unwrap(); + cipher + .decrypt(&nonce, &aad, &oracle_combined, &mut opened) + .expect("RustCrypto ChaCha20-Poly1305 ciphertext must open in rscrypto"); assert_eq!( opened, plaintext, "combined decrypt mismatch pt_len={plaintext_len} aad_len={aad_len}" @@ -268,14 +286,16 @@ fn chacha20poly1305_boundary_and_large_inputs_match_oracle() { aad: &aad, }, ) - .unwrap(); + .expect("rscrypto ChaCha20-Poly1305 ciphertext must open in RustCrypto"); assert_eq!( oracle_opened, plaintext, "oracle decrypt mismatch pt_len={plaintext_len} aad_len={aad_len}" ); let mut detached = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut detached).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut detached) + .expect("rscrypto detached ChaCha20-Poly1305 oracle input must seal"); assert_eq!( detached, oracle_combined[..plaintext_len], @@ -290,7 +310,7 @@ fn chacha20poly1305_boundary_and_large_inputs_match_oracle() { let typed_tag = ChaCha20Poly1305Tag::from_bytes(tag.to_bytes()); cipher .decrypt_in_place(&nonce, &aad, &mut detached, &typed_tag) - .unwrap(); + .expect("fresh detached ChaCha20-Poly1305 ciphertext must authenticate"); assert_eq!( detached, plaintext, "detached decrypt mismatch pt_len={plaintext_len} aad_len={aad_len}" diff --git a/tests/common/array.rs b/tests/common/array.rs new file mode 100644 index 00000000..c647a0a0 --- /dev/null +++ b/tests/common/array.rs @@ -0,0 +1,10 @@ +//! Fixed-size hexadecimal test-vector decoding. + +/// Decode a hex string into a fixed-size byte array. +/// +/// Panics if the decoded length does not equal `N`. +pub(crate) fn decode_hex_array(hex: &str) -> [u8; N] { + crate::common::decode_hex_vec(hex) + .try_into() + .expect("hex length must match the destination array") +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 005b5b10..00950740 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -3,30 +3,24 @@ /// Decode a hex string into a `Vec`. /// /// Panics on odd-length input or invalid hex characters. -#[allow(dead_code)] -pub fn decode_hex_vec(hex: &str) -> Vec { +pub(crate) fn decode_hex_vec(hex: &str) -> Vec { assert_eq!(hex.len() % 2, 0, "hex length must be even"); hex .as_bytes() .chunks_exact(2) - .map(|pair| (nibble(pair[0]) << 4) | nibble(pair[1])) + .map(|pair| { + let high = nibble(pair[0]).expect("hex input must contain only hexadecimal digits"); + let low = nibble(pair[1]).expect("hex input must contain only hexadecimal digits"); + (high << 4) | low + }) .collect() } -/// Decode a hex string into a fixed-size byte array. -/// -/// Panics if the decoded length does not equal `N`. -#[allow(dead_code)] -pub fn decode_hex_array(hex: &str) -> [u8; N] { - let v = decode_hex_vec(hex); - v.try_into().expect("hex length does not match array size") -} - -fn nibble(b: u8) -> u8 { +fn nibble(b: u8) -> Option { match b { - b'0'..=b'9' => b - b'0', - b'a'..=b'f' => b - b'a' + 10, - b'A'..=b'F' => b - b'A' + 10, - _ => panic!("invalid hex byte: {b:#04x}"), + b'0'..=b'9' => Some(b.strict_sub(b'0')), + b'a'..=b'f' => Some(b.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(b.strict_sub(b'A').strict_add(10)), + _ => None, } } diff --git a/tests/common_properties.rs b/tests/common_properties.rs index a5fa9b82..1f971712 100644 --- a/tests/common_properties.rs +++ b/tests/common_properties.rs @@ -25,6 +25,18 @@ use rscrypto::{Checksum, ChecksumCombine, Crc16Ccitt, Crc16Ibm, Crc24OpenPgp, Cr const REF_CRC24_OPENPGP: RefCrc = RefCrc::::new(&crc::CRC_24_OPENPGP); +fn reference_u16(value: u64) -> u16 { + u16::try_from(value).expect("CRC-16 reference output must fit in 16 bits") +} + +fn reference_u32(value: u64) -> u32 { + u32::try_from(value).expect("CRC-32 reference output must fit in 32 bits") +} + +fn low_byte(value: usize) -> u8 { + value.to_le_bytes()[0] +} + // Combine Correctness Tests // // These tests prove the fundamental combine property: @@ -52,7 +64,7 @@ proptest! { let combined = Crc16Ccitt::combine(crc_a, crc_b, b.len()); // Reference: independent crc-fast implementation over full data - let expected = crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, &data) as u16; + let expected = reference_u16(crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, &data)); prop_assert_eq!(combined, expected, "combine(crc(A), crc(B), len(B)) != crc(A||B) at split {}/{}", @@ -71,7 +83,7 @@ proptest! { let crc_b = Crc16Ibm::checksum(b); let combined = Crc16Ibm::combine(crc_a, crc_b, b.len()); - let expected = crc_fast::checksum(CrcAlgorithm::Crc16Arc, &data) as u16; + let expected = reference_u16(crc_fast::checksum(CrcAlgorithm::Crc16Arc, &data)); prop_assert_eq!(combined, expected, "combine(crc(A), crc(B), len(B)) != crc(A||B) at split {}/{}", @@ -113,7 +125,7 @@ proptest! { let crc_b = Crc32::checksum(b); let combined = Crc32::combine(crc_a, crc_b, b.len()); - let expected = crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, &data) as u32; + let expected = reference_u32(crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, &data)); prop_assert_eq!(combined, expected, "combine(crc(A), crc(B), len(B)) != crc(A||B) at split {}/{}", @@ -132,7 +144,7 @@ proptest! { let crc_b = Crc32C::checksum(b); let combined = Crc32C::combine(crc_a, crc_b, b.len()); - let expected = crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, &data) as u32; + let expected = reference_u32(crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, &data)); prop_assert_eq!(combined, expected, "combine(crc(A), crc(B), len(B)) != crc(A||B) at split {}/{}", @@ -210,7 +222,7 @@ fn apply_chunking(data: &[u8], chunk_pattern: &[usize]) -> C::Outpu let end = (offset.strict_add(chunk_size)).min(data.len()); hasher.update(&data[offset..end]); offset = end; - pattern_idx = pattern_idx.strict_add(1) % chunk_pattern.len(); + pattern_idx = pattern_idx.strict_add(1).strict_rem(chunk_pattern.len()); } hasher.finalize() @@ -234,7 +246,7 @@ fn apply_chunking_vectored(data: &[u8], chunk_pattern: &[usize]) -> let end = (offset.strict_add(chunk_size)).min(data.len()); chunks.push(&data[offset..end]); offset = end; - pattern_idx = pattern_idx.strict_add(1) % chunk_pattern.len(); + pattern_idx = pattern_idx.strict_add(1).strict_rem(chunk_pattern.len()); } hasher.update_vectored(&chunks); @@ -352,7 +364,7 @@ macro_rules! test_combine_all_splits { // Test various small sizes including edge cases for size in [0, 1, 2, 3, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 255, 256] { - let data: Vec = (0..size).map(|i| (i as u8).wrapping_mul(17)).collect(); + let data: Vec = (0..size).map(|i| low_byte(i).wrapping_mul(17)).collect(); for split in 0..=data.len() { let (a, b) = data.split_at(split); @@ -379,19 +391,19 @@ macro_rules! test_combine_all_splits { } test_combine_all_splits!(crc16_ccitt_combine_all_splits, Crc16Ccitt, |data: &[u8]| { - crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, data) as u16 + reference_u16(crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, data)) }); test_combine_all_splits!(crc16_ibm_combine_all_splits, Crc16Ibm, |data: &[u8]| { - crc_fast::checksum(CrcAlgorithm::Crc16Arc, data) as u16 + reference_u16(crc_fast::checksum(CrcAlgorithm::Crc16Arc, data)) }); test_combine_all_splits!(crc24_openpgp_combine_all_splits, Crc24OpenPgp, |data: &[u8]| { REF_CRC24_OPENPGP.checksum(data) & 0x00FF_FFFF }); test_combine_all_splits!(crc32_ieee_combine_all_splits, Crc32, |data: &[u8]| { - crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, data) as u32 + reference_u32(crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, data)) }); test_combine_all_splits!(crc32c_combine_all_splits, Crc32C, |data: &[u8]| { - crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, data) as u32 + reference_u32(crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, data)) }); test_combine_all_splits!(crc64_xz_combine_all_splits, Crc64, |data: &[u8]| { crc_fast::checksum(CrcAlgorithm::Crc64Xz, data) @@ -409,7 +421,7 @@ macro_rules! test_chunking_edge_cases { for size in [ 0, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, 256, 512, 1024, ] { - let data: Vec = (0..size).map(|i| (i as u8).wrapping_mul(23)).collect(); + let data: Vec = (0..size).map(|i| low_byte(i).wrapping_mul(23)).collect(); let expected = <$crc_type>::checksum(&data); // Byte-at-a-time diff --git a/tests/crc16_properties.rs b/tests/crc16_properties.rs index 8a869309..16f3afe7 100644 --- a/tests/crc16_properties.rs +++ b/tests/crc16_properties.rs @@ -11,20 +11,24 @@ use crc_fast::CrcAlgorithm; use proptest::prelude::*; use rscrypto::{Checksum, ChecksumCombine, Crc16Ccitt, Crc16Ibm}; +fn reference_u16(value: u64) -> u16 { + u16::try_from(value).expect("CRC-16 reference output must fit in 16 bits") +} + proptest! { // Cross-validation against crc-fast-rust #[test] fn crc16_ccitt_matches_crc_fast_rust(data in proptest::collection::vec(any::(), 0..=4096)) { let ours = Crc16Ccitt::checksum(&data); - let reference = crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, &data) as u16; + let reference = reference_u16(crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, &data)); prop_assert_eq!(ours, reference); } #[test] fn crc16_ibm_matches_crc_fast_rust(data in proptest::collection::vec(any::(), 0..=4096)) { let ours = Crc16Ibm::checksum(&data); - let reference = crc_fast::checksum(CrcAlgorithm::Crc16Arc, &data) as u16; + let reference = reference_u16(crc_fast::checksum(CrcAlgorithm::Crc16Arc, &data)); prop_assert_eq!(ours, reference); } @@ -41,7 +45,7 @@ proptest! { reference.update(part); } - prop_assert_eq!(ours.finalize(), reference.finalize() as u16); + prop_assert_eq!(ours.finalize(), reference_u16(reference.finalize())); } #[test] @@ -57,7 +61,7 @@ proptest! { reference.update(part); } - prop_assert_eq!(ours.finalize(), reference.finalize() as u16); + prop_assert_eq!(ours.finalize(), reference_u16(reference.finalize())); } #[test] @@ -74,8 +78,12 @@ proptest! { let ref_crc_a = crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, a); let ref_crc_b = crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, b); - let ref_combined = - crc_fast::checksum_combine(CrcAlgorithm::Crc16IbmSdlc, ref_crc_a, ref_crc_b, b.len() as u64) as u16; + let ref_combined = reference_u16(crc_fast::checksum_combine( + CrcAlgorithm::Crc16IbmSdlc, + ref_crc_a, + ref_crc_b, + u64::try_from(b.len()).expect("test input length must fit in u64"), + )); prop_assert_eq!(combined, ref_combined); } @@ -94,8 +102,12 @@ proptest! { let ref_crc_a = crc_fast::checksum(CrcAlgorithm::Crc16Arc, a); let ref_crc_b = crc_fast::checksum(CrcAlgorithm::Crc16Arc, b); - let ref_combined = - crc_fast::checksum_combine(CrcAlgorithm::Crc16Arc, ref_crc_a, ref_crc_b, b.len() as u64) as u16; + let ref_combined = reference_u16(crc_fast::checksum_combine( + CrcAlgorithm::Crc16Arc, + ref_crc_a, + ref_crc_b, + u64::try_from(b.len()).expect("test input length must fit in u64"), + )); prop_assert_eq!(combined, ref_combined); } diff --git a/tests/crc32_properties.rs b/tests/crc32_properties.rs index 864dd25f..4ba575b2 100644 --- a/tests/crc32_properties.rs +++ b/tests/crc32_properties.rs @@ -11,20 +11,24 @@ use crc_fast::CrcAlgorithm; use proptest::prelude::*; use rscrypto::{Checksum, ChecksumCombine, Crc32, Crc32C}; +fn reference_u32(value: u64) -> u32 { + u32::try_from(value).expect("CRC-32 reference output must fit in 32 bits") +} + proptest! { // Cross-validation against crc-fast-rust #[test] fn crc32_matches_crc_fast_rust(data in proptest::collection::vec(any::(), 0..=4096)) { let ours = Crc32::checksum(&data); - let reference = crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, &data) as u32; + let reference = reference_u32(crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, &data)); prop_assert_eq!(ours, reference); } #[test] fn crc32c_matches_crc_fast_rust(data in proptest::collection::vec(any::(), 0..=4096)) { let ours = Crc32C::checksum(&data); - let reference = crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, &data) as u32; + let reference = reference_u32(crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, &data)); prop_assert_eq!(ours, reference); } @@ -38,7 +42,7 @@ proptest! { reference.update(part); } - prop_assert_eq!(ours.finalize(), reference.finalize() as u32); + prop_assert_eq!(ours.finalize(), reference_u32(reference.finalize())); } #[test] @@ -51,7 +55,7 @@ proptest! { reference.update(part); } - prop_assert_eq!(ours.finalize(), reference.finalize() as u32); + prop_assert_eq!(ours.finalize(), reference_u32(reference.finalize())); } #[test] @@ -65,7 +69,12 @@ proptest! { let ref_crc_a = crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, a); let ref_crc_b = crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, b); - let ref_combined = crc_fast::checksum_combine(CrcAlgorithm::Crc32IsoHdlc, ref_crc_a, ref_crc_b, b.len() as u64) as u32; + let ref_combined = reference_u32(crc_fast::checksum_combine( + CrcAlgorithm::Crc32IsoHdlc, + ref_crc_a, + ref_crc_b, + u64::try_from(b.len()).expect("test input length must fit in u64"), + )); prop_assert_eq!(combined, ref_combined); } @@ -81,7 +90,12 @@ proptest! { let ref_crc_a = crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, a); let ref_crc_b = crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, b); - let ref_combined = crc_fast::checksum_combine(CrcAlgorithm::Crc32Iscsi, ref_crc_a, ref_crc_b, b.len() as u64) as u32; + let ref_combined = reference_u32(crc_fast::checksum_combine( + CrcAlgorithm::Crc32Iscsi, + ref_crc_a, + ref_crc_b, + u64::try_from(b.len()).expect("test input length must fit in u64"), + )); prop_assert_eq!(combined, ref_combined); } diff --git a/tests/cshake256_differential.rs b/tests/cshake256_differential.rs index 02cd04f6..98e26668 100644 --- a/tests/cshake256_differential.rs +++ b/tests/cshake256_differential.rs @@ -22,9 +22,19 @@ fn cshake256_ref(function_name: &[u8], customization: &[u8], data: &[u8], out: & fn decode_hex_64(value: &str) -> [u8; 64] { assert_eq!(value.len(), 128); let mut out = [0u8; 64]; - for (index, byte) in out.iter_mut().enumerate() { - let offset = index * 2; - *byte = u8::from_str_radix(&value[offset..offset + 2], 16).unwrap(); + let mut digits = value.chars(); + for byte in &mut out { + let high = digits + .next() + .expect("cSHAKE vector must contain 128 hexadecimal digits"); + let low = digits.next().expect("cSHAKE vector must contain complete byte pairs"); + let high = high + .to_digit(16) + .expect("cSHAKE vector must contain only hexadecimal digits"); + let low = low + .to_digit(16) + .expect("cSHAKE vector must contain only hexadecimal digits"); + *byte = u8::try_from(high.strict_mul(16).strict_add(low)).expect("two hexadecimal digits must fit in one byte"); } out } @@ -59,8 +69,8 @@ proptest! { let mut hasher = Cshake128::new(&function_name, &customization); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); hasher.update(&data[i..end]); i = end; } @@ -99,8 +109,8 @@ proptest! { let mut hasher = Cshake256::new(&function_name, &customization); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); hasher.update(&data[i..end]); i = end; } diff --git a/tests/cshake256_nist_vectors.rs b/tests/cshake256_nist_vectors.rs index c9e603d4..9e8f66e8 100644 --- a/tests/cshake256_nist_vectors.rs +++ b/tests/cshake256_nist_vectors.rs @@ -6,7 +6,9 @@ mod common; use common::decode_hex_vec as decode_hex; fn ascending_bytes(len: usize) -> Vec { - (0..len).map(|byte| byte as u8).collect() + (0..len) + .map(|byte| u8::try_from(byte).expect("cSHAKE fixture byte must fit in u8")) + .collect() } #[test] diff --git a/tests/ecdsa_oracle.rs b/tests/ecdsa_oracle.rs index 4c6c14b5..46571f7d 100644 --- a/tests/ecdsa_oracle.rs +++ b/tests/ecdsa_oracle.rs @@ -23,19 +23,24 @@ fn array_from_slice(slice: &[u8]) -> [u8; N] { fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("short DER length must fit in one byte")]; } let bytes = len.to_be_bytes(); - let first = bytes.iter().position(|&byte| byte != 0).unwrap_or(bytes.len() - 1); + let first = bytes + .iter() + .position(|&byte| byte != 0) + .expect("long DER length must contain a nonzero byte"); let body = &bytes[first..]; - let mut out = Vec::with_capacity(body.len() + 1); - out.push(0x80 | (body.len() as u8)); + let mut out = Vec::with_capacity(body.len().strict_add(1)); + let body_len = u8::try_from(body.len()).expect("DER length-of-length must fit in one byte"); + out.push(0x80 | body_len); out.extend_from_slice(body); out } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + value.len() + 4); + let capacity = value.len().strict_add(5); + let mut out = Vec::with_capacity(capacity); out.push(tag); out.extend_from_slice(&der_len(value.len())); out.extend_from_slice(value); @@ -47,7 +52,7 @@ fn spki_der(curve_oid: &[u8], sec1: &[u8]) -> Vec { algorithm.extend_from_slice(&tlv(0x06, ID_EC_PUBLIC_KEY_OID)); algorithm.extend_from_slice(&tlv(0x06, curve_oid)); - let mut bit_string = Vec::with_capacity(sec1.len() + 1); + let mut bit_string = Vec::with_capacity(sec1.len().strict_add(1)); bit_string.push(0); bit_string.extend_from_slice(sec1); @@ -75,8 +80,12 @@ fn p256_verify_accepts_rustcrypto_raw_and_der_signatures() { .expect("P-256 raw signature must parse"); let der = EcdsaP256Signature::from_der(oracle_signature.to_der().as_bytes()).expect("P-256 DER signature must parse"); - assert!(public.verify(message, &raw).is_ok()); - assert!(public.verify(message, &der).is_ok()); + public + .verify(message, &raw) + .expect("rscrypto must verify the RustCrypto P-256 raw signature"); + public + .verify(message, &der) + .expect("rscrypto must verify the RustCrypto P-256 DER signature"); } #[cfg(feature = "ecdsa-p256")] @@ -97,15 +106,14 @@ fn p256_verify_rejects_tampered_rustcrypto_signature() { bytes[17] ^= 0x40; let tampered = EcdsaP256Signature::from_bytes(bytes).expect("tampered P-256 signature scalar shape must parse"); - assert!(public.verify(message, &tampered).is_err()); - assert!( - public - .verify( - b"wrong message", - &EcdsaP256Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())).unwrap() - ) - .is_err() - ); + public + .verify(message, &tampered) + .expect_err("rscrypto must reject a tampered P-256 signature"); + let original = EcdsaP256Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())) + .expect("original P-256 signature must parse"); + public + .verify(b"wrong message", &original) + .expect_err("rscrypto must reject a P-256 signature for the wrong message"); } #[cfg(feature = "ecdsa-p256")] @@ -123,7 +131,9 @@ fn p256_sign_derives_oracle_public_key_and_signature_verifies_with_rustcrypto() let oracle_signature = P256OracleSignature::from_slice(rs_signature.as_bytes()).expect("P-256 oracle signature must parse"); - assert!(rs_public.verify(message, &rs_signature).is_ok()); + rs_public + .verify(message, &rs_signature) + .expect("rscrypto must verify its P-256 signature"); p256::ecdsa::signature::Verifier::verify(verifying_key, message, &oracle_signature) .expect("RustCrypto must verify rscrypto P-256 signature"); } @@ -150,7 +160,9 @@ fn p256_blinded_sign_matches_deterministic_signature_and_rustcrypto_oracle() { P256OracleSignature::from_slice(blinded.as_bytes()).expect("P-256 oracle signature must parse"); assert_eq!(deterministic, blinded); - assert!(rs_public.verify(message, &blinded).is_ok()); + rs_public + .verify(message, &blinded) + .expect("rscrypto must verify its blinded P-256 signature"); p256::ecdsa::signature::Verifier::verify(&oracle_public, message, &oracle_signature) .expect("RustCrypto must verify rscrypto blinded P-256 signature"); } @@ -173,8 +185,12 @@ fn p384_verify_accepts_rustcrypto_raw_and_der_signatures() { .expect("P-384 raw signature must parse"); let der = EcdsaP384Signature::from_der(oracle_signature.to_der().as_bytes()).expect("P-384 DER signature must parse"); - assert!(public.verify(message, &raw).is_ok()); - assert!(public.verify(message, &der).is_ok()); + public + .verify(message, &raw) + .expect("rscrypto must verify the RustCrypto P-384 raw signature"); + public + .verify(message, &der) + .expect("rscrypto must verify the RustCrypto P-384 DER signature"); } #[cfg(feature = "ecdsa-p384")] @@ -195,15 +211,14 @@ fn p384_verify_rejects_tampered_rustcrypto_signature() { bytes[29] ^= 0x10; let tampered = EcdsaP384Signature::from_bytes(bytes).expect("tampered P-384 signature scalar shape must parse"); - assert!(public.verify(message, &tampered).is_err()); - assert!( - public - .verify( - b"wrong message", - &EcdsaP384Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())).unwrap() - ) - .is_err() - ); + public + .verify(message, &tampered) + .expect_err("rscrypto must reject a tampered P-384 signature"); + let original = EcdsaP384Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())) + .expect("original P-384 signature must parse"); + public + .verify(b"wrong message", &original) + .expect_err("rscrypto must reject a P-384 signature for the wrong message"); } #[cfg(feature = "ecdsa-p384")] @@ -221,7 +236,9 @@ fn p384_sign_derives_oracle_public_key_and_signature_verifies_with_rustcrypto() let oracle_signature = P384OracleSignature::from_slice(rs_signature.as_bytes()).expect("P-384 oracle signature must parse"); - assert!(rs_public.verify(message, &rs_signature).is_ok()); + rs_public + .verify(message, &rs_signature) + .expect("rscrypto must verify its P-384 signature"); p384::ecdsa::signature::Verifier::verify(verifying_key, message, &oracle_signature) .expect("RustCrypto must verify rscrypto P-384 signature"); } @@ -248,7 +265,9 @@ fn p384_blinded_sign_matches_deterministic_signature_and_rustcrypto_oracle() { P384OracleSignature::from_slice(blinded.as_bytes()).expect("P-384 oracle signature must parse"); assert_eq!(deterministic, blinded); - assert!(rs_public.verify(message, &blinded).is_ok()); + rs_public + .verify(message, &blinded) + .expect("rscrypto must verify its blinded P-384 signature"); p384::ecdsa::signature::Verifier::verify(&oracle_public, message, &oracle_signature) .expect("RustCrypto must verify rscrypto blinded P-384 signature"); } diff --git a/tests/ecdsa_wycheproof.rs b/tests/ecdsa_wycheproof.rs index 9de609cc..acfa75f4 100644 --- a/tests/ecdsa_wycheproof.rs +++ b/tests/ecdsa_wycheproof.rs @@ -29,9 +29,11 @@ struct Suite<'a> { } fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] + value + .get(name) + .expect("Wycheproof field must be present") .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + .expect("Wycheproof field must be a string") } fn assert_der_vectors( @@ -47,7 +49,10 @@ fn assert_der_vectors( assert_eq!(suite["schema"].as_str(), Some("ecdsa_verify_schema_v1.json")); assert_eq!( suite["numberOfTests"].as_u64(), - Some((spec.expected.valid + spec.expected.invalid) as u64) + Some( + u64::try_from(spec.expected.valid.strict_add(spec.expected.invalid)) + .expect("Wycheproof test count must fit in u64") + ) ); let groups = suite["testGroups"] @@ -70,7 +75,12 @@ fn assert_der_vectors( let signature = decode_hex_vec(field(test, "sig")); let verified = verify_der(&public, &message, &signature); - match field(test, "result") { + let disposition = field(test, "result"); + assert!( + matches!(disposition, "valid" | "invalid"), + "unsupported Wycheproof ECDSA result `{disposition}`" + ); + match disposition { "valid" => { counts.valid = counts.valid.strict_add(1); assert!( @@ -89,7 +99,7 @@ fn assert_der_vectors( field(test, "comment") ); } - other => panic!("unsupported Wycheproof ECDSA result `{other}`"), + _ => {} } } } diff --git a/tests/ed25519_oracle.rs b/tests/ed25519_oracle.rs index 4e89f2bb..238d15ab 100644 --- a/tests/ed25519_oracle.rs +++ b/tests/ed25519_oracle.rs @@ -5,7 +5,7 @@ use rscrypto::{Ed25519Keypair, Ed25519PublicKey, Ed25519SecretKey, Ed25519Signat fn patterned_message(len: usize, mul: u8, add: u8) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(mul).wrapping_add(add)) + .map(|i| i.to_le_bytes()[0].wrapping_mul(mul).wrapping_add(add)) .collect() } @@ -32,12 +32,18 @@ fn ed25519_matches_dalek_for_deterministic_cases() { assert_eq!(public.to_bytes(), verifying_key.to_bytes()); assert_eq!(ours.to_bytes(), oracle.to_bytes()); - assert!(public.verify(&message, &ours).is_ok()); + public + .verify(&message, &ours) + .expect("rscrypto signature must verify with its public key"); let oracle_public = Ed25519PublicKey::from_bytes(verifying_key.to_bytes()); let oracle_signature = Ed25519Signature::from_bytes(oracle.to_bytes()); - assert!(oracle_public.verify(&message, &oracle_signature).is_ok()); - assert!(verifying_key.verify_strict(&message, &oracle).is_ok()); + oracle_public + .verify(&message, &oracle_signature) + .expect("Dalek signature must verify through rscrypto"); + verifying_key + .verify_strict(&message, &oracle) + .expect("Dalek signature must verify through Dalek"); } } @@ -56,6 +62,10 @@ fn ed25519_and_dalek_agree_on_rejection_for_tampered_signature() { let ours = Ed25519Signature::from_bytes(signature); let oracle = DalekSignature::from_bytes(&signature); - assert!(public.verify(&message, &ours).is_err()); - assert!(verifying_key.verify_strict(&message, &oracle).is_err()); + public + .verify(&message, &ours) + .expect_err("rscrypto must reject a tampered signature"); + verifying_key + .verify_strict(&message, &oracle) + .expect_err("Dalek must reject a tampered signature"); } diff --git a/tests/ed25519_rfc8032_vectors.rs b/tests/ed25519_rfc8032_vectors.rs index 36a7d1ca..a50b66ff 100644 --- a/tests/ed25519_rfc8032_vectors.rs +++ b/tests/ed25519_rfc8032_vectors.rs @@ -12,7 +12,10 @@ struct Vector<'a> { } mod common; -use common::{decode_hex_array as decode_hex, decode_hex_vec}; +#[path = "common/array.rs"] +mod hex_array; +use common::decode_hex_vec; +use hex_array::decode_hex_array as decode_hex; #[test] fn ed25519_rfc8032_vectors() { diff --git a/tests/ed25519_wycheproof.rs b/tests/ed25519_wycheproof.rs index 29c66bc7..7b395a4a 100644 --- a/tests/ed25519_wycheproof.rs +++ b/tests/ed25519_wycheproof.rs @@ -4,7 +4,10 @@ use rscrypto::{Ed25519PublicKey, Ed25519Signature}; use serde_json::Value; mod common; -use common::{decode_hex_array, decode_hex_vec}; +#[path = "common/array.rs"] +mod hex_array; +use common::decode_hex_vec; +use hex_array::decode_hex_array; const ED25519: &str = include_str!("../testdata/auth/wycheproof/ed25519_test.json"); @@ -15,9 +18,11 @@ struct Counts { } fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] + value + .get(name) + .expect("Wycheproof field must be present") .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + .expect("Wycheproof field must be a string") } fn groups(suite: &Value) -> &[Value] { @@ -45,7 +50,12 @@ fn wycheproof_ed25519_verify_vectors_match_expected_results() { let message = decode_hex_vec(field(test, "msg")); let signature = decode_hex_vec(field(test, "sig")); - match field(test, "result") { + let disposition = field(test, "result"); + assert!( + matches!(disposition, "valid" | "invalid"), + "unsupported Wycheproof Ed25519 result `{disposition}`" + ); + match disposition { "valid" => { counts.valid = counts.valid.strict_add(1); let signature = Ed25519Signature::from_bytes( @@ -53,25 +63,24 @@ fn wycheproof_ed25519_verify_vectors_match_expected_results() { .try_into() .expect("valid Wycheproof Ed25519 signatures must be 64 bytes"), ); - assert!( - public.verify(&message, &signature).is_ok(), - "Wycheproof Ed25519 tcId {} rejected a valid signature", - test["tcId"] - ); + public + .verify(&message, &signature) + .expect("Wycheproof valid Ed25519 signature must verify"); } "invalid" => { counts.invalid = counts.invalid.strict_add(1); if signature.len() == Ed25519Signature::LENGTH { - let signature = Ed25519Signature::from_bytes(signature.try_into().unwrap()); - assert!( - public.verify(&message, &signature).is_err(), - "Wycheproof Ed25519 tcId {} accepted an invalid signature: {}", - test["tcId"], - field(test, "comment") + let signature = Ed25519Signature::from_bytes( + signature + .try_into() + .expect("length-checked Wycheproof Ed25519 signature must fit an array"), ); + public + .verify(&message, &signature) + .expect_err("Wycheproof invalid Ed25519 signature must be rejected"); } } - other => panic!("unsupported Wycheproof Ed25519 result `{other}`"), + _ => {} } } } diff --git a/tests/fast_hash_allocations.rs b/tests/fast_hash_allocations.rs index d36eccef..fc2cdb37 100644 --- a/tests/fast_hash_allocations.rs +++ b/tests/fast_hash_allocations.rs @@ -18,14 +18,16 @@ std::thread_local! { } fn record_allocation() { - let _ = ALLOCATIONS.try_with(|allocations| { + discard_measurement_result(ALLOCATIONS.try_with(|allocations| { let count = allocations.get(); if count != MEASUREMENT_DISABLED { allocations.set(count.strict_add(1)); } - }); + })); } +fn discard_measurement_result(_result: Result<(), std::thread::AccessError>) {} + // SAFETY: Delegating allocation to `System` because: // 1. Every operation forwards the original pointer and layout unchanged. // 2. `System` defines and upholds the global allocator contract. @@ -66,7 +68,7 @@ struct AllocationMeasurement; impl Drop for AllocationMeasurement { fn drop(&mut self) { - let _ = ALLOCATIONS.try_with(|allocations| allocations.set(MEASUREMENT_DISABLED)); + discard_measurement_result(ALLOCATIONS.try_with(|allocations| allocations.set(MEASUREMENT_DISABLED))); } } @@ -99,7 +101,8 @@ fn fast_hashers_and_preallocated_maps_hash_without_allocating() { for chunk in long_input.chunks(127) { xxh3.write(chunk); } - let _ = xxh3.finish(); + let hash = xxh3.finish(); + assert_eq!(xxh3.finish(), hash); }); let xxh3_128_direct = measure_allocations(|| { @@ -107,13 +110,15 @@ fn fast_hashers_and_preallocated_maps_hash_without_allocating() { for chunk in long_input.chunks(127) { xxh3_128.write(chunk); } - let _ = xxh3_128.finish(); + let hash = xxh3_128.finish(); + assert_eq!(xxh3_128.finish(), hash); }); let rapid_direct = measure_allocations(|| { let mut rapid = rapid_builder.build_hasher(); rapid.write(&long_input); - let _ = rapid.finish(); + let hash = rapid.finish(); + assert_eq!(rapid.finish(), hash); }); let rapid_stream_direct = measure_allocations(|| { @@ -121,17 +126,18 @@ fn fast_hashers_and_preallocated_maps_hash_without_allocating() { for chunk in long_input.chunks(127) { rapid_stream.write(chunk); } - let _ = rapid_stream.finish(); + let hash = rapid_stream.finish(); + assert_eq!(rapid_stream.finish(), hash); }); let xxh3_map_ops = measure_allocations(|| { xxh3_map.insert("allocation-free", 1); - let _ = xxh3_map.get("allocation-free"); + assert_eq!(xxh3_map.get("allocation-free"), Some(&1)); }); let rapid_map_ops = measure_allocations(|| { rapid_map.insert("allocation-free", 1); - let _ = rapid_map.get("allocation-free"); + assert_eq!(rapid_map.get("allocation-free"), Some(&1)); }); assert_eq!(xxh3_direct, 0, "XXH3 Hasher must not allocate"); diff --git a/tests/getrandom_smoke.rs b/tests/getrandom_smoke.rs index 870c9147..471dc0e7 100644 --- a/tests/getrandom_smoke.rs +++ b/tests/getrandom_smoke.rs @@ -5,13 +5,23 @@ #![cfg(feature = "getrandom")] -#[allow(unused_macros)] +#[cfg(any( + feature = "chacha20poly1305", + feature = "xchacha20poly1305", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "aegis256", + feature = "aead", + feature = "ed25519", + feature = "x25519" +))] macro_rules! random_smoke { ($name:ident, $ty:ty) => { #[test] fn $name() { - let a = <$ty>::try_random().unwrap(); - let b = <$ty>::try_random().unwrap(); + let a = <$ty>::try_random().expect("OS entropy must produce the first value"); + let b = <$ty>::try_random().expect("OS entropy must produce the second value"); // Must not be all-zero (overwhelmingly unlikely from a CSPRNG). assert!(a.as_bytes().iter().any(|&b| b != 0), "try_random() returned all zeros"); @@ -89,7 +99,7 @@ mod ed25519_rng { #[test] fn keypair_try_generate() { - let keypair = Ed25519Keypair::try_generate().unwrap(); + let keypair = Ed25519Keypair::try_generate().expect("OS entropy must generate an Ed25519 keypair"); assert!(keypair.secret_key().as_bytes().iter().any(|&b| b != 0)); } } @@ -101,7 +111,7 @@ mod x25519_rng { #[test] fn secret_key_try_generate() { - let secret = X25519SecretKey::try_generate().unwrap(); + let secret = X25519SecretKey::try_generate().expect("OS entropy must generate an X25519 secret key"); assert!(secret.as_bytes().iter().any(|&b| b != 0)); } } @@ -114,8 +124,8 @@ mod rapidhash_rng { #[test] fn random_state_uses_platform_entropy() { - let first = RapidRandomState::try_new().unwrap(); - let second = RapidRandomState::try_new().unwrap(); + let first = RapidRandomState::try_new().expect("OS entropy must initialize the first random state"); + let second = RapidRandomState::try_new().expect("OS entropy must initialize the second random state"); assert_ne!( first.hash_one(b"collection key"), second.hash_one(b"collection key"), @@ -130,7 +140,7 @@ mod ecdsa_p256_rng { #[test] fn secret_key_try_generate() { - let secret = EcdsaP256SecretKey::try_generate().unwrap(); + let secret = EcdsaP256SecretKey::try_generate().expect("OS entropy must generate a P-256 secret key"); assert!(secret.as_bytes().iter().any(|&b| b != 0)); } } @@ -141,7 +151,7 @@ mod ecdsa_p384_rng { #[test] fn secret_key_try_generate() { - let secret = EcdsaP384SecretKey::try_generate().unwrap(); + let secret = EcdsaP384SecretKey::try_generate().expect("OS entropy must generate a P-384 secret key"); assert!(secret.as_bytes().iter().any(|&b| b != 0)); } } @@ -152,9 +162,9 @@ mod mlkem_rng { #[test] fn keypair_try_generate() { - let (ek512, dk512) = MlKem512::try_generate_keypair().unwrap(); - let (ek768, dk768) = MlKem768::try_generate_keypair().unwrap(); - let (ek1024, dk1024) = MlKem1024::try_generate_keypair().unwrap(); + let (ek512, dk512) = MlKem512::try_generate_keypair().expect("OS entropy must generate an ML-KEM-512 keypair"); + let (ek768, dk768) = MlKem768::try_generate_keypair().expect("OS entropy must generate an ML-KEM-768 keypair"); + let (ek1024, dk1024) = MlKem1024::try_generate_keypair().expect("OS entropy must generate an ML-KEM-1024 keypair"); assert!(ek512.as_bytes().iter().any(|&b| b != 0)); assert!(dk512.as_bytes().iter().any(|&b| b != 0)); diff --git a/tests/hkdf_sha256_vectors.rs b/tests/hkdf_sha256_vectors.rs index d19d755c..a1af3ac8 100644 --- a/tests/hkdf_sha256_vectors.rs +++ b/tests/hkdf_sha256_vectors.rs @@ -3,10 +3,12 @@ use rscrypto::{HkdfSha256, auth::HkdfOutputLengthError}; mod common; +#[path = "common/array.rs"] +mod hex_array; use common::decode_hex_vec; fn decode_hex_array(hex: &str) -> [u8; N] { - common::decode_hex_array(&hex.replace('\n', "")) + hex_array::decode_hex_array(&hex.replace('\n', "")) } #[test] @@ -21,7 +23,9 @@ fn hkdf_sha256_rfc5869_case_1() { &decode_hex_array::<32>("077709362c2e32df0ddc3f0dc47bba6390b6c73bb50f9c3122ec844ad7c2b3e5") ); - let okm = hkdf.expand_array::<42>(&info).unwrap(); + let okm = hkdf + .expand_array::<42>(&info) + .expect("RFC 5869 HKDF-SHA-256 case 1 expansion must succeed"); assert_eq!( okm, decode_hex_array::<42>( @@ -33,7 +37,8 @@ fn hkdf_sha256_rfc5869_case_1() { #[test] fn hkdf_sha256_rfc5869_case_3() { - let okm = HkdfSha256::derive_array::<42>(b"", &[0x0b; 22], b"").unwrap(); + let okm = HkdfSha256::derive_array::<42>(b"", &[0x0b; 22], b"") + .expect("RFC 5869 HKDF-SHA-256 case 3 derivation must succeed"); assert_eq!( okm, decode_hex_array::<42>( @@ -45,7 +50,8 @@ fn hkdf_sha256_rfc5869_case_3() { #[test] fn hkdf_sha256_rejects_oversized_output() { - let mut out = vec![0u8; HkdfSha256::MAX_OUTPUT_SIZE + 1]; - let err = HkdfSha256::derive(b"salt", b"ikm", b"info", &mut out).unwrap_err(); + let mut out = vec![0u8; HkdfSha256::MAX_OUTPUT_SIZE.strict_add(1)]; + let err = + HkdfSha256::derive(b"salt", b"ikm", b"info", &mut out).expect_err("HKDF-SHA-256 must reject oversized output"); assert_eq!(err, HkdfOutputLengthError::new()); } diff --git a/tests/hkdf_sha384_vectors.rs b/tests/hkdf_sha384_vectors.rs index 53068622..0ad18327 100644 --- a/tests/hkdf_sha384_vectors.rs +++ b/tests/hkdf_sha384_vectors.rs @@ -4,7 +4,10 @@ use hkdf::Hkdf as RustCryptoHkdf; use rscrypto::{HkdfSha384, auth::HkdfOutputLengthError}; mod common; -use common::{decode_hex_array, decode_hex_vec}; +#[path = "common/array.rs"] +mod hex_array; +use common::decode_hex_vec; +use hex_array::decode_hex_array; #[test] fn hkdf_sha384_case_1() { @@ -20,7 +23,9 @@ fn hkdf_sha384_case_1() { ) ); - let okm = hkdf.expand_array::<42>(&info).unwrap(); + let okm = hkdf + .expand_array::<42>(&info) + .expect("HKDF-SHA-384 case 1 expansion must succeed"); assert_eq!( okm, decode_hex_array::<42>("9b5097a86038b805309076a44b3a9f38063e25b516dcbf369f394cfab43685f748b6457763e4f0204fc5",) @@ -53,7 +58,9 @@ fn hkdf_sha384_case_2() { ) ); - let okm = hkdf.expand_array::<82>(&info).unwrap(); + let okm = hkdf + .expand_array::<82>(&info) + .expect("HKDF-SHA-384 case 2 expansion must succeed"); assert_eq!( okm, decode_hex_array::<82>( @@ -65,7 +72,7 @@ fn hkdf_sha384_case_2() { #[test] fn hkdf_sha384_case_3() { - let okm = HkdfSha384::derive_array::<42>(b"", &[0x0b; 22], b"").unwrap(); + let okm = HkdfSha384::derive_array::<42>(b"", &[0x0b; 22], b"").expect("HKDF-SHA-384 case 3 derivation must succeed"); assert_eq!( okm, decode_hex_array::<42>("c8c96e710f89b0d7990bca68bcdec8cf854062e54c73a7abc743fade9b242daacc1cea5670415b52849c",) @@ -84,8 +91,12 @@ fn hkdf_sha384_matches_rustcrypto() { let mut ours = vec![0u8; len]; let mut theirs = vec![0u8; len]; - hkdf.expand(&info, &mut ours).unwrap(); - rustcrypto.expand(&info, &mut theirs).unwrap(); + hkdf + .expand(&info, &mut ours) + .expect("rscrypto HKDF-SHA-384 differential expansion must succeed"); + rustcrypto + .expand(&info, &mut theirs) + .expect("RustCrypto HKDF-SHA-384 differential expansion must succeed"); assert_eq!(ours, theirs, "HKDF-SHA384 mismatch at output len {len}"); } @@ -98,13 +109,17 @@ fn hkdf_sha384_derive_matches_extract_then_expand() { let info = b"context"; let extracted = HkdfSha384::new(salt, ikm); - let derived = HkdfSha384::derive_array::<96>(salt, ikm, info).unwrap(); - assert_eq!(derived, extracted.expand_array::<96>(info).unwrap()); + let derived = HkdfSha384::derive_array::<96>(salt, ikm, info).expect("HKDF-SHA-384 one-shot derivation must succeed"); + let expanded = extracted + .expand_array::<96>(info) + .expect("HKDF-SHA-384 extracted-state expansion must succeed"); + assert_eq!(derived, expanded); } #[test] fn hkdf_sha384_rejects_oversized_output() { - let mut out = vec![0u8; HkdfSha384::MAX_OUTPUT_SIZE + 1]; - let err = HkdfSha384::derive(b"salt", b"ikm", b"info", &mut out).unwrap_err(); + let mut out = vec![0u8; HkdfSha384::MAX_OUTPUT_SIZE.strict_add(1)]; + let err = + HkdfSha384::derive(b"salt", b"ikm", b"info", &mut out).expect_err("HKDF-SHA-384 must reject oversized output"); assert_eq!(err, HkdfOutputLengthError::new()); } diff --git a/tests/hkdf_sha512_vectors.rs b/tests/hkdf_sha512_vectors.rs index 0ae5c5da..da71b8b6 100644 --- a/tests/hkdf_sha512_vectors.rs +++ b/tests/hkdf_sha512_vectors.rs @@ -4,7 +4,10 @@ use hkdf::Hkdf as RustCryptoHkdf; use rscrypto::{HkdfSha512, auth::HkdfOutputLengthError}; mod common; -use common::{decode_hex_array, decode_hex_vec}; +#[path = "common/array.rs"] +mod hex_array; +use common::decode_hex_vec; +use hex_array::decode_hex_array; #[test] fn hkdf_sha512_rfc5869_case_1() { @@ -21,7 +24,9 @@ fn hkdf_sha512_rfc5869_case_1() { ) ); - let okm = hkdf.expand_array::<42>(&info).unwrap(); + let okm = hkdf + .expand_array::<42>(&info) + .expect("HKDF-SHA-512 vector expansion must succeed"); assert_eq!( okm, decode_hex_array::<42>("832390086cda71fb47625bb5ceB168e4c8e26a1a16ed34d9fc7fe92c1481579338da362cb8d9f925d7cb",) @@ -40,8 +45,12 @@ fn hkdf_sha512_matches_rustcrypto() { let mut ours = vec![0u8; len]; let mut theirs = vec![0u8; len]; - hkdf.expand(&info, &mut ours).unwrap(); - rustcrypto.expand(&info, &mut theirs).unwrap(); + hkdf + .expand(&info, &mut ours) + .expect("rscrypto HKDF-SHA-512 differential expansion must succeed"); + rustcrypto + .expand(&info, &mut theirs) + .expect("RustCrypto HKDF-SHA-512 differential expansion must succeed"); assert_eq!(ours, theirs, "HKDF-SHA512 mismatch at output len {len}"); } @@ -54,13 +63,18 @@ fn hkdf_sha512_derive_matches_extract_then_expand() { let info = b"context"; let extracted = HkdfSha512::new(salt, ikm); - let derived = HkdfSha512::derive_array::<128>(salt, ikm, info).unwrap(); - assert_eq!(derived, extracted.expand_array::<128>(info).unwrap()); + let derived = + HkdfSha512::derive_array::<128>(salt, ikm, info).expect("HKDF-SHA-512 one-shot derivation must succeed"); + let expanded = extracted + .expand_array::<128>(info) + .expect("HKDF-SHA-512 extracted-state expansion must succeed"); + assert_eq!(derived, expanded); } #[test] fn hkdf_sha512_rejects_oversized_output() { - let mut out = vec![0u8; HkdfSha512::MAX_OUTPUT_SIZE + 1]; - let err = HkdfSha512::derive(b"salt", b"ikm", b"info", &mut out).unwrap_err(); + let mut out = vec![0u8; HkdfSha512::MAX_OUTPUT_SIZE.strict_add(1)]; + let err = + HkdfSha512::derive(b"salt", b"ikm", b"info", &mut out).expect_err("HKDF-SHA-512 must reject oversized output"); assert_eq!(err, HkdfOutputLengthError::new()); } diff --git a/tests/hkdf_wycheproof.rs b/tests/hkdf_wycheproof.rs index da17f963..126ca489 100644 --- a/tests/hkdf_wycheproof.rs +++ b/tests/hkdf_wycheproof.rs @@ -9,7 +9,7 @@ use common::decode_hex_vec; const HKDF_SHA256: &str = include_str!("../testdata/auth/wycheproof/hkdf_sha256_test.json"); const HKDF_SHA384: &str = include_str!("../testdata/auth/wycheproof/hkdf_sha384_test.json"); -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] struct Counts { valid: usize, invalid: usize, @@ -18,9 +18,24 @@ struct Counts { type HkdfDerive = fn(&[u8], &[u8], &[u8], &mut [u8]) -> Result<(), HkdfOutputLengthError>; fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + value + .get(name) + .and_then(Value::as_str) + .expect("Wycheproof string field must exist and contain a string") +} + +enum VectorResult { + Valid, + Invalid, +} + +fn vector_result(test: &Value) -> VectorResult { + let result = match field(test, "result") { + "valid" => Some(VectorResult::Valid), + "invalid" => Some(VectorResult::Invalid), + _ => None, + }; + result.expect("HKDF Wycheproof result must be valid or invalid") } fn groups(suite: &Value) -> &[Value] { @@ -43,25 +58,22 @@ fn run_hkdf_suite(suite_json: &str, algorithm: &str, derive: HkdfDerive, expecte let ikm = decode_hex_vec(field(test, "ikm")); let salt = decode_hex_vec(field(test, "salt")); let info = decode_hex_vec(field(test, "info")); - let size = test["size"].as_u64().expect("size must be numeric") as usize; + let size = usize::try_from(test["size"].as_u64().expect("size must be numeric")) + .expect("HKDF Wycheproof size must fit in usize"); let mut okm = vec![0u8; size]; - match field(test, "result") { - "valid" => { - counts.valid += 1; + match vector_result(test) { + VectorResult::Valid => { + counts.valid = counts.valid.strict_add(1); let expected_okm = decode_hex_vec(field(test, "okm")); assert_eq!(expected_okm.len(), size, "{algorithm} tcId {tc_id} okm length mismatch"); - derive(&salt, &ikm, &info, &mut okm).unwrap_or_else(|err| panic!("{algorithm} tcId {tc_id} failed: {err}")); + derive(&salt, &ikm, &info, &mut okm).expect("known-valid HKDF Wycheproof vector must derive"); assert_eq!(okm, expected_okm, "{algorithm} tcId {tc_id} OKM mismatch"); } - "invalid" => { - counts.invalid += 1; - assert!( - derive(&salt, &ikm, &info, &mut okm).is_err(), - "{algorithm} tcId {tc_id} accepted invalid size" - ); + VectorResult::Invalid => { + counts.invalid = counts.invalid.strict_add(1); + derive(&salt, &ikm, &info, &mut okm).expect_err("HKDF must reject an invalid Wycheproof output size"); } - other => panic!("{algorithm} tcId {tc_id} has unsupported result `{other}`"), } } } diff --git a/tests/hmac_sha256_proptest.rs b/tests/hmac_sha256_proptest.rs index e747e7e6..0088c73a 100644 --- a/tests/hmac_sha256_proptest.rs +++ b/tests/hmac_sha256_proptest.rs @@ -14,7 +14,8 @@ proptest! { let ours = HmacSha256::mac(&key, &data); - let mut oracle = Hmac::::new_from_slice(&key).unwrap(); + let mut oracle = Hmac::::new_from_slice(&key) + .expect("RustCrypto HMAC-SHA-256 must accept every generated key length"); oracle.update(&data); let expected = oracle.finalize().into_bytes(); diff --git a/tests/hmac_sha256_vectors.rs b/tests/hmac_sha256_vectors.rs index 1cf2ed8d..5217c179 100644 --- a/tests/hmac_sha256_vectors.rs +++ b/tests/hmac_sha256_vectors.rs @@ -3,7 +3,9 @@ use rscrypto::{HmacSha256, HmacSha256Tag}; mod common; -use common::decode_hex_array as decode_hex; +#[path = "common/array.rs"] +mod hex_array; +use hex_array::decode_hex_array as decode_hex; #[test] fn hmac_sha256_rfc4231_vectors() { @@ -37,7 +39,7 @@ fn hmac_sha256_rfc4231_vectors() { actual.ct_eq(&expected).declassify(), "HMAC-SHA256 RFC 4231 vector {i} mismatch" ); - assert!(HmacSha256::verify_tag(key, data, &expected).is_ok()); + HmacSha256::verify_tag(key, data, &expected).expect("HMAC-SHA-256 must verify an RFC 4231 tag"); } } @@ -51,10 +53,7 @@ fn hmac_sha256_verify_rejects_corrupted_tag() { let mut corrupted = tag.to_bytes(); corrupted[index] ^= 0x80; let corrupted = HmacSha256Tag::from_bytes(corrupted); - assert!( - HmacSha256::verify_tag(key, data, &corrupted).is_err(), - "HMAC-SHA256 accepted a tag corrupted at byte {index}" - ); + HmacSha256::verify_tag(key, data, &corrupted).expect_err("HMAC-SHA-256 must reject a corrupted full-length tag"); } } @@ -64,14 +63,13 @@ fn hmac_sha256_verifies_rfc4231_tag_truncated_to_64_bits() { let data = b"Hi There"; let expected = decode_hex::<8>("b0344c61d8db3853"); - assert!(HmacSha256::verify_truncated_tag_64(&key, data, &expected).is_ok()); + HmacSha256::verify_truncated_tag_64(&key, data, &expected) + .expect("HMAC-SHA-256 must verify the RFC 4231 truncated tag"); for index in 0..expected.len() { let mut corrupted = expected; corrupted[index] ^= 0x80; - assert!( - HmacSha256::verify_truncated_tag_64(&key, data, &corrupted).is_err(), - "HMAC-SHA256 accepted a 64-bit tag corrupted at byte {index}" - ); + HmacSha256::verify_truncated_tag_64(&key, data, &corrupted) + .expect_err("HMAC-SHA-256 must reject a corrupted 64-bit tag"); } } diff --git a/tests/hmac_sha2_family_vectors.rs b/tests/hmac_sha2_family_vectors.rs index 4abe584b..78429ec9 100644 --- a/tests/hmac_sha2_family_vectors.rs +++ b/tests/hmac_sha2_family_vectors.rs @@ -3,7 +3,9 @@ use rscrypto::{HmacSha384, HmacSha384Tag, HmacSha512, HmacSha512Tag, Mac}; mod common; -use common::decode_hex_array as decode_hex; +#[path = "common/array.rs"] +mod hex_array; +use hex_array::decode_hex_array as decode_hex; #[test] fn hmac_sha384_rfc4231_vectors() { @@ -50,7 +52,7 @@ fn hmac_sha384_rfc4231_vectors() { actual.ct_eq(&expected).declassify(), "HMAC-SHA384 RFC 4231 vector {i} mismatch" ); - assert!(HmacSha384::verify_tag(key, data, &expected).is_ok()); + HmacSha384::verify_tag(key, data, &expected).expect("HMAC-SHA-384 must verify an RFC 4231 tag"); } let truncated = HmacSha384::mac(&[0x0c; 20], b"Test With Truncation"); @@ -105,7 +107,7 @@ fn hmac_sha512_rfc4231_vectors() { actual.ct_eq(&expected).declassify(), "HMAC-SHA512 RFC 4231 vector {i} mismatch" ); - assert!(HmacSha512::verify_tag(key, data, &expected).is_ok()); + HmacSha512::verify_tag(key, data, &expected).expect("HMAC-SHA-512 must verify an RFC 4231 tag"); } let truncated = HmacSha512::mac(&[0x0c; 20], b"Test With Truncation"); @@ -125,10 +127,7 @@ fn hmac_sha384_verify_rejects_corrupted_tag_positions() { let mut corrupted = tag.to_bytes(); corrupted[index] ^= 0x80; let corrupted = HmacSha384Tag::from_bytes(corrupted); - assert!( - HmacSha384::verify_tag(key, data, &corrupted).is_err(), - "HMAC-SHA384 accepted a tag corrupted at byte {index}" - ); + HmacSha384::verify_tag(key, data, &corrupted).expect_err("HMAC-SHA-384 must reject a corrupted tag"); } } @@ -142,10 +141,7 @@ fn hmac_sha512_verify_rejects_corrupted_tag_positions() { let mut corrupted = tag.to_bytes(); corrupted[index] ^= 0x80; let corrupted = HmacSha512Tag::from_bytes(corrupted); - assert!( - HmacSha512::verify_tag(key, data, &corrupted).is_err(), - "HMAC-SHA512 accepted a tag corrupted at byte {index}" - ); + HmacSha512::verify_tag(key, data, &corrupted).expect_err("HMAC-SHA-512 must reject a corrupted tag"); } } diff --git a/tests/hmac_sha384_proptest.rs b/tests/hmac_sha384_proptest.rs index ae121f29..661cb699 100644 --- a/tests/hmac_sha384_proptest.rs +++ b/tests/hmac_sha384_proptest.rs @@ -14,7 +14,8 @@ proptest! { let ours = HmacSha384::mac(&key, &data); - let mut oracle = Hmac::::new_from_slice(&key).unwrap(); + let mut oracle = Hmac::::new_from_slice(&key) + .expect("RustCrypto HMAC-SHA-384 must accept every generated key length"); oracle.update(&data); let expected = oracle.finalize().into_bytes(); diff --git a/tests/hmac_sha3_vectors.rs b/tests/hmac_sha3_vectors.rs index 49b083df..1b570c2b 100644 --- a/tests/hmac_sha3_vectors.rs +++ b/tests/hmac_sha3_vectors.rs @@ -9,9 +9,9 @@ use sha3::Digest as _; fn pattern(len: usize, mul: u8, add: u8) -> Vec { (0..len) .map(|i| { - (i as u8) + i.to_le_bytes()[0] .wrapping_mul(mul) - .wrapping_add(((i >> 3) as u8).wrapping_add(add)) + .wrapping_add((i >> 3).to_le_bytes()[0].wrapping_add(add)) }) .collect() } @@ -71,7 +71,7 @@ macro_rules! assert_hmac_sha3 { key_len, data_len ); - assert!(<$ours>::verify_tag(&key, &data, &expected).is_ok()); + <$ours>::verify_tag(&key, &data, &expected).expect("HMAC-SHA3 must verify the independent-oracle tag"); let mut streaming = <$ours>::new(&key); for chunk in data.chunks(chunk_len) { @@ -98,7 +98,8 @@ macro_rules! assert_hmac_sha3 { if !corrupted.is_empty() { corrupted[corrupted.len() / 2] ^= 0x80; } - assert!(<$ours>::verify_tag(&key, &data, &<$tag>::from_bytes(corrupted)).is_err()); + <$ours>::verify_tag(&key, &data, &<$tag>::from_bytes(corrupted)) + .expect_err("HMAC-SHA3 must reject a corrupted tag"); } }}; } diff --git a/tests/hmac_sha512_proptest.rs b/tests/hmac_sha512_proptest.rs index 0352f94a..edad7773 100644 --- a/tests/hmac_sha512_proptest.rs +++ b/tests/hmac_sha512_proptest.rs @@ -14,7 +14,8 @@ proptest! { let ours = HmacSha512::mac(&key, &data); - let mut oracle = Hmac::::new_from_slice(&key).unwrap(); + let mut oracle = Hmac::::new_from_slice(&key) + .expect("RustCrypto HMAC-SHA-512 must accept every generated key length"); oracle.update(&data); let expected = oracle.finalize().into_bytes(); diff --git a/tests/hmac_wycheproof.rs b/tests/hmac_wycheproof.rs index 748d5575..4761854f 100644 --- a/tests/hmac_wycheproof.rs +++ b/tests/hmac_wycheproof.rs @@ -10,16 +10,31 @@ const HMAC_SHA256: &str = include_str!("../testdata/auth/wycheproof/hmac_sha256_ const HMAC_SHA384: &str = include_str!("../testdata/auth/wycheproof/hmac_sha384_test.json"); const HMAC_SHA512: &str = include_str!("../testdata/auth/wycheproof/hmac_sha512_test.json"); -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] struct Counts { valid: usize, invalid: usize, } fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + value + .get(name) + .and_then(Value::as_str) + .expect("Wycheproof string field must exist and contain a string") +} + +enum VectorResult { + Valid, + Invalid, +} + +fn vector_result(test: &Value) -> VectorResult { + let result = match field(test, "result") { + "valid" => Some(VectorResult::Valid), + "invalid" => Some(VectorResult::Invalid), + _ => None, + }; + result.expect("HMAC Wycheproof result must be valid or invalid") } fn groups(suite: &Value) -> &[Value] { @@ -55,34 +70,26 @@ fn run_hmac_suite( let key = decode_hex_vec(field(test, "key")); let msg = decode_hex_vec(field(test, "msg")); let tag = decode_hex_vec(field(test, "tag")); - let tag_bytes: [u8; TAG_SIZE] = match tag.try_into() { - Ok(tag) => tag, - Err(tag) => panic!("{algorithm} tcId {tc_id} tag has wrong length: {}", tag.len()), - }; + let tag_bytes: [u8; TAG_SIZE] = tag + .try_into() + .expect("full-tag HMAC Wycheproof vector must have the algorithm tag length"); let tag = M::Tag::from(tag_bytes); - match field(test, "result") { - "valid" => { - counts.valid += 1; + match vector_result(test) { + VectorResult::Valid => { + counts.valid = counts.valid.strict_add(1); let actual = M::mac(&key, &msg); assert_eq!( >::as_ref(&actual), >::as_ref(&tag), "{algorithm} tcId {tc_id} MAC mismatch" ); - assert!( - M::verify_tag(&key, &msg, &tag).is_ok(), - "{algorithm} tcId {tc_id} verify failed" - ); + M::verify_tag(&key, &msg, &tag).expect("HMAC must verify a known-valid Wycheproof tag"); } - "invalid" => { - counts.invalid += 1; - assert!( - M::verify_tag(&key, &msg, &tag).is_err(), - "{algorithm} tcId {tc_id} accepted an invalid tag" - ); + VectorResult::Invalid => { + counts.invalid = counts.invalid.strict_add(1); + M::verify_tag(&key, &msg, &tag).expect_err("HMAC must reject a known-invalid Wycheproof tag"); } - other => panic!("{algorithm} tcId {tc_id} has unsupported result `{other}`"), } } } diff --git a/tests/kmac128_differential.rs b/tests/kmac128_differential.rs index 4a6e7750..a4014240 100644 --- a/tests/kmac128_differential.rs +++ b/tests/kmac128_differential.rs @@ -12,21 +12,37 @@ fn kmac128_ref(key: &[u8], customization: &[u8], data: &[u8], out: &mut [u8]) { } fn encoded_string_len(len: usize) -> usize { - let bits = len * 8; - let width = ((usize::BITS - bits.leading_zeros()) as usize).div_ceil(8).max(1); - 1 + width + len + let bits = len.strict_mul(8); + let width_bits = usize::BITS.strict_sub(bits.leading_zeros()); + let width = usize::try_from(width_bits) + .expect("encoded-string width must fit usize") + .div_ceil(8) + .max(1); + 1usize.strict_add(width).strict_add(len) } fn bytepad_is_aligned(rate: usize, segments: &[usize]) -> bool { - (2 + segments.iter().map(|&len| encoded_string_len(len)).sum::()).is_multiple_of(rate) + 2usize + .strict_add(segments.iter().map(|&len| encoded_string_len(len)).sum::()) + .is_multiple_of(rate) } fn decode_hex_32(value: &str) -> [u8; 32] { assert_eq!(value.len(), 64); let mut out = [0u8; 32]; - for (index, byte) in out.iter_mut().enumerate() { - let offset = index * 2; - *byte = u8::from_str_radix(&value[offset..offset + 2], 16).unwrap(); + let mut digits = value.chars(); + for byte in &mut out { + let high = digits + .next() + .expect("KMAC-128 vector must contain 64 hexadecimal digits"); + let low = digits.next().expect("KMAC-128 vector must contain complete byte pairs"); + let high = high + .to_digit(16) + .expect("KMAC-128 vector must contain only hexadecimal digits"); + let low = low + .to_digit(16) + .expect("KMAC-128 vector must contain only hexadecimal digits"); + *byte = u8::try_from(high.strict_mul(16).strict_add(low)).expect("two hexadecimal digits must fit in one byte"); } out } @@ -47,7 +63,7 @@ proptest! { let mut actual = vec![0u8; out_len]; Kmac128::mac_into(&key, &customization, &data, &mut actual); - prop_assert_eq!(actual, expected.clone()); + prop_assert_eq!(actual.as_slice(), expected.as_slice()); if expected.is_empty() { prop_assert!(Kmac128::verify_tag(&key, &customization, &data, &expected).is_err()); } else { @@ -77,8 +93,8 @@ proptest! { let mut kmac = Kmac128::new(&key, &customization); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); kmac.update(&data[i..end]); i = end; } diff --git a/tests/kmac128_nist_vectors.rs b/tests/kmac128_nist_vectors.rs index a9d9e2ec..7a8545e5 100644 --- a/tests/kmac128_nist_vectors.rs +++ b/tests/kmac128_nist_vectors.rs @@ -6,7 +6,9 @@ mod common; use common::decode_hex_vec as decode_hex; fn ascending_bytes(len: usize) -> Vec { - (0..len).map(|byte| byte as u8).collect() + (0..len) + .map(|byte| u8::try_from(byte).expect("NIST KMAC-128 sample index must fit in one byte")) + .collect() } fn key_bytes() -> Vec { @@ -22,7 +24,7 @@ fn kmac128_nist_sample_1_matches() { let mut actual = vec![0u8; expected.len()]; Kmac128::mac_into(&key, b"", &data, &mut actual); assert_eq!(actual, expected, "kmac128 sample 1 one-shot mismatch"); - assert!(Kmac128::verify_tag(&key, b"", &data, &expected).is_ok()); + Kmac128::verify_tag(&key, b"", &data, &expected).expect("KMAC-128 must verify NIST sample 1"); } #[test] @@ -34,7 +36,7 @@ fn kmac128_nist_sample_2_matches() { let mut actual = vec![0u8; expected.len()]; Kmac128::mac_into(&key, b"My Tagged Application", &data, &mut actual); assert_eq!(actual, expected, "kmac128 sample 2 one-shot mismatch"); - assert!(Kmac128::verify_tag(&key, b"My Tagged Application", &data, &expected).is_ok()); + Kmac128::verify_tag(&key, b"My Tagged Application", &data, &expected).expect("KMAC-128 must verify NIST sample 2"); } #[test] @@ -46,7 +48,7 @@ fn kmac128_nist_sample_3_matches() { let mut actual = vec![0u8; expected.len()]; Kmac128::mac_into(&key, b"My Tagged Application", &data, &mut actual); assert_eq!(actual, expected, "kmac128 sample 3 one-shot mismatch"); - assert!(Kmac128::verify_tag(&key, b"My Tagged Application", &data, &expected).is_ok()); + Kmac128::verify_tag(&key, b"My Tagged Application", &data, &expected).expect("KMAC-128 must verify NIST sample 3"); let mut streaming = Kmac128::new(&key, b"My Tagged Application"); streaming.update(&data[..168]); @@ -62,17 +64,13 @@ fn kmac128_verify_rejects_empty_and_corrupted_tags() { let data = ascending_bytes(200); let expected = Kmac128::mac_array::<32>(&key, b"My Tagged Application", &data); - assert!( - Kmac128::verify_tag(&key, b"My Tagged Application", &data, &[]).is_err(), - "KMAC128 must reject an empty expected tag" - ); + Kmac128::verify_tag(&key, b"My Tagged Application", &data, &[]) + .expect_err("KMAC-128 must reject an empty expected tag"); for index in [0, expected.len() / 2, expected.len() - 1] { let mut corrupted = expected; corrupted[index] ^= 0x80; - assert!( - Kmac128::verify_tag(&key, b"My Tagged Application", &data, &corrupted).is_err(), - "KMAC128 accepted a tag corrupted at byte {index}" - ); + Kmac128::verify_tag(&key, b"My Tagged Application", &data, &corrupted) + .expect_err("KMAC-128 must reject a corrupted tag"); } } diff --git a/tests/kmac256_differential.rs b/tests/kmac256_differential.rs index 707d14b3..d21b457b 100644 --- a/tests/kmac256_differential.rs +++ b/tests/kmac256_differential.rs @@ -12,21 +12,37 @@ fn kmac256_ref(key: &[u8], customization: &[u8], data: &[u8], out: &mut [u8]) { } fn encoded_string_len(len: usize) -> usize { - let bits = len * 8; - let width = ((usize::BITS - bits.leading_zeros()) as usize).div_ceil(8).max(1); - 1 + width + len + let bits = len.strict_mul(8); + let width_bits = usize::BITS.strict_sub(bits.leading_zeros()); + let width = usize::try_from(width_bits) + .expect("encoded-string width must fit usize") + .div_ceil(8) + .max(1); + 1usize.strict_add(width).strict_add(len) } fn bytepad_is_aligned(rate: usize, segments: &[usize]) -> bool { - (2 + segments.iter().map(|&len| encoded_string_len(len)).sum::()).is_multiple_of(rate) + 2usize + .strict_add(segments.iter().map(|&len| encoded_string_len(len)).sum::()) + .is_multiple_of(rate) } fn decode_hex_64(value: &str) -> [u8; 64] { assert_eq!(value.len(), 128); let mut out = [0u8; 64]; - for (index, byte) in out.iter_mut().enumerate() { - let offset = index * 2; - *byte = u8::from_str_radix(&value[offset..offset + 2], 16).unwrap(); + let mut digits = value.chars(); + for byte in &mut out { + let high = digits + .next() + .expect("KMAC-256 vector must contain 128 hexadecimal digits"); + let low = digits.next().expect("KMAC-256 vector must contain complete byte pairs"); + let high = high + .to_digit(16) + .expect("KMAC-256 vector must contain only hexadecimal digits"); + let low = low + .to_digit(16) + .expect("KMAC-256 vector must contain only hexadecimal digits"); + *byte = u8::try_from(high.strict_mul(16).strict_add(low)).expect("two hexadecimal digits must fit in one byte"); } out } @@ -47,7 +63,7 @@ proptest! { let mut actual = vec![0u8; out_len]; Kmac256::mac_into(&key, &customization, &data, &mut actual); - prop_assert_eq!(actual, expected.clone()); + prop_assert_eq!(actual.as_slice(), expected.as_slice()); if expected.is_empty() { prop_assert!(Kmac256::verify_tag(&key, &customization, &data, &expected).is_err()); } else { @@ -77,8 +93,8 @@ proptest! { let mut kmac = Kmac256::new(&key, &customization); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); kmac.update(&data[i..end]); i = end; } diff --git a/tests/kmac256_nist_vectors.rs b/tests/kmac256_nist_vectors.rs index 35e1d95b..87b24585 100644 --- a/tests/kmac256_nist_vectors.rs +++ b/tests/kmac256_nist_vectors.rs @@ -6,7 +6,9 @@ mod common; use common::decode_hex_vec as decode_hex; fn ascending_bytes(len: usize) -> Vec { - (0..len).map(|byte| byte as u8).collect() + (0..len) + .map(|byte| u8::try_from(byte).expect("NIST KMAC-256 sample index must fit in one byte")) + .collect() } fn key_bytes() -> Vec { @@ -25,7 +27,7 @@ fn kmac256_nist_sample_4_matches() { let mut actual = vec![0u8; expected.len()]; Kmac256::mac_into(&key, b"My Tagged Application", &data, &mut actual); assert_eq!(actual, expected, "kmac256 sample 4 one-shot mismatch"); - assert!(Kmac256::verify_tag(&key, b"My Tagged Application", &data, &expected).is_ok()); + Kmac256::verify_tag(&key, b"My Tagged Application", &data, &expected).expect("KMAC-256 must verify NIST sample 4"); let mut streaming = Kmac256::new(&key, b"My Tagged Application"); streaming.update(&data[..1]); @@ -47,7 +49,7 @@ fn kmac256_nist_sample_5_matches() { let mut actual = vec![0u8; expected.len()]; Kmac256::mac_into(&key, b"", &data, &mut actual); assert_eq!(actual, expected, "kmac256 sample 5 one-shot mismatch"); - assert!(Kmac256::verify_tag(&key, b"", &data, &expected).is_ok()); + Kmac256::verify_tag(&key, b"", &data, &expected).expect("KMAC-256 must verify NIST sample 5"); let mut streaming = Kmac256::new(&key, b""); streaming.update(&data[..136]); @@ -69,7 +71,7 @@ fn kmac256_nist_sample_6_matches() { let mut actual = vec![0u8; expected.len()]; Kmac256::mac_into(&key, b"My Tagged Application", &data, &mut actual); assert_eq!(actual, expected, "kmac256 sample 6 one-shot mismatch"); - assert!(Kmac256::verify_tag(&key, b"My Tagged Application", &data, &expected).is_ok()); + Kmac256::verify_tag(&key, b"My Tagged Application", &data, &expected).expect("KMAC-256 must verify NIST sample 6"); let mut streaming = Kmac256::new(&key, b"My Tagged Application"); streaming.update(&data[..136]); @@ -85,23 +87,17 @@ fn kmac256_verify_rejects_empty_and_corrupted_tags() { let data = ascending_bytes(200); let expected = Kmac256::mac_array::<64>(&key, b"My Tagged Application", &data); - assert!( - Kmac256::verify_tag(&key, b"My Tagged Application", &data, &[]).is_err(), - "KMAC256 must reject an empty expected tag" - ); + Kmac256::verify_tag(&key, b"My Tagged Application", &data, &[]) + .expect_err("KMAC-256 must reject an empty expected tag"); let one_byte = [expected[0] ^ 0x80]; - assert!( - Kmac256::verify_tag(&key, b"My Tagged Application", &data, &one_byte).is_err(), - "KMAC256 accepted a corrupted one-byte tag" - ); + Kmac256::verify_tag(&key, b"My Tagged Application", &data, &one_byte) + .expect_err("KMAC-256 must reject a corrupted one-byte tag"); for index in [0, expected.len() / 2, expected.len() - 1] { let mut corrupted = expected; corrupted[index] ^= 0x80; - assert!( - Kmac256::verify_tag(&key, b"My Tagged Application", &data, &corrupted).is_err(), - "KMAC256 accepted a tag corrupted at byte {index}" - ); + Kmac256::verify_tag(&key, b"My Tagged Application", &data, &corrupted) + .expect_err("KMAC-256 must reject a corrupted full-length tag"); } } diff --git a/tests/kmac_wycheproof.rs b/tests/kmac_wycheproof.rs index 2b43d65b..210d4720 100644 --- a/tests/kmac_wycheproof.rs +++ b/tests/kmac_wycheproof.rs @@ -8,16 +8,31 @@ use common::decode_hex_vec; const KMAC256_NO_CUSTOMIZATION: &str = include_str!("../testdata/auth/wycheproof/kmac256_no_customization_test.json"); -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] struct Counts { valid: usize, invalid: usize, } fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + value + .get(name) + .and_then(Value::as_str) + .expect("Wycheproof string field must exist and contain a string") +} + +enum VectorResult { + Valid, + Invalid, +} + +fn vector_result(test: &Value) -> VectorResult { + let result = match field(test, "result") { + "valid" => Some(VectorResult::Valid), + "invalid" => Some(VectorResult::Invalid), + _ => None, + }; + result.expect("KMAC Wycheproof result must be valid or invalid") } fn groups(suite: &Value) -> &[Value] { @@ -42,25 +57,20 @@ fn kmac256_no_customization_wycheproof_vectors() { let msg = decode_hex_vec(field(test, "msg")); let tag = decode_hex_vec(field(test, "tag")); - match field(test, "result") { - "valid" => { - counts.valid += 1; + match vector_result(test) { + VectorResult::Valid => { + counts.valid = counts.valid.strict_add(1); let mut actual = vec![0u8; tag.len()]; Kmac256::mac_into(&key, b"", &msg, &mut actual); assert_eq!(actual, tag, "KMAC256 tcId {tc_id} tag mismatch"); - assert!( - Kmac256::verify_tag_primitive(&key, b"", &msg, &tag).is_ok(), - "KMAC256 tcId {tc_id} verify failed" - ); + Kmac256::verify_tag_primitive(&key, b"", &msg, &tag) + .expect("KMAC256 must verify a known-valid Wycheproof tag"); } - "invalid" => { - counts.invalid += 1; - assert!( - Kmac256::verify_tag_primitive(&key, b"", &msg, &tag).is_err(), - "KMAC256 tcId {tc_id} accepted an invalid tag" - ); + VectorResult::Invalid => { + counts.invalid = counts.invalid.strict_add(1); + Kmac256::verify_tag_primitive(&key, b"", &msg, &tag) + .expect_err("KMAC256 must reject a known-invalid Wycheproof tag"); } - other => panic!("KMAC256 tcId {tc_id} has unsupported result `{other}`"), } } } @@ -77,5 +87,5 @@ fn kmac256_no_customization_wycheproof_vectors() { #[test] fn kmac256_rejects_empty_verification_tag() { - assert!(Kmac256::verify_tag(b"key", b"", b"message", b"").is_err()); + Kmac256::verify_tag(b"key", b"", b"message", b"").expect_err("KMAC256 must reject an empty verification tag"); } diff --git a/tests/migration_aws_lc_rs.rs b/tests/migration_aws_lc_rs.rs index 6c5d5254..ec6efb39 100644 --- a/tests/migration_aws_lc_rs.rs +++ b/tests/migration_aws_lc_rs.rs @@ -62,15 +62,17 @@ fn test_aws_lc_rs_digest_hmac_hkdf_and_pbkdf2_migration_examples_are_byte_equiva aws_hkdf::Salt::new(aws_hkdf::HKDF_SHA256, salt) .extract(ikm) .expand(&[info], AwsHkdfLen(aws_okm.len())) - .unwrap() + .expect("42-byte AWS-LC HKDF expansion must be valid") .fill(&mut aws_okm) - .unwrap(); + .expect("AWS-LC HKDF output buffer must match the requested length"); let mut ours_okm = [0u8; 42]; - HkdfSha256::new(salt, ikm).expand(info, &mut ours_okm).unwrap(); + HkdfSha256::new(salt, ikm) + .expand(info, &mut ours_okm) + .expect("42-byte rscrypto HKDF expansion must be valid"); assert_eq!(ours_okm, aws_okm); - let iterations = NonZeroU32::new(600_000).unwrap(); + let iterations = NonZeroU32::new(600_000).expect("PBKDF2 iteration count must be nonzero"); let mut aws_pbkdf2 = [0u8; 32]; aws_pbkdf2::derive( aws_pbkdf2::PBKDF2_HMAC_SHA256, @@ -81,7 +83,8 @@ fn test_aws_lc_rs_digest_hmac_hkdf_and_pbkdf2_migration_examples_are_byte_equiva ); let mut ours_pbkdf2 = [0u8; 32]; - Pbkdf2Sha256::derive_key(b"migration password", salt, iterations.get(), &mut ours_pbkdf2).unwrap(); + Pbkdf2Sha256::derive_key(b"migration password", salt, iterations.get(), &mut ours_pbkdf2) + .expect("rscrypto PBKDF2 parameters must be valid"); assert_eq!(ours_pbkdf2, aws_pbkdf2); } @@ -91,21 +94,29 @@ fn test_aws_lc_rs_aead_migration_examples_are_byte_equivalent() { let aes = Aes256Gcm::new(&Aes256GcmKey::from_bytes(KEY_32)); let nonce = Nonce96::from_bytes(NONCE_12); let mut ours_aes = vec![0u8; DATA.len() + 16]; - aes.encrypt(&nonce, AAD, DATA, &mut ours_aes).unwrap(); + aes + .encrypt(&nonce, AAD, DATA, &mut ours_aes) + .expect("rscrypto AES-256-GCM seal buffer must fit plaintext and tag"); assert_eq!(ours_aes, aws_aes); let mut opened = vec![0u8; DATA.len()]; - aes.decrypt(&nonce, AAD, &ours_aes, &mut opened).unwrap(); + aes + .decrypt(&nonce, AAD, &ours_aes, &mut opened) + .expect("freshly sealed rscrypto AES-256-GCM ciphertext must authenticate"); assert_eq!(opened, DATA); let aws_chacha = aws_aead_seal(&aws_aead::CHACHA20_POLY1305, &KEY_32, DATA); let chacha = ChaCha20Poly1305::new(&ChaCha20Poly1305Key::from_bytes(KEY_32)); let mut ours_chacha = vec![0u8; DATA.len() + 16]; - chacha.encrypt(&nonce, AAD, DATA, &mut ours_chacha).unwrap(); + chacha + .encrypt(&nonce, AAD, DATA, &mut ours_chacha) + .expect("rscrypto ChaCha20-Poly1305 seal buffer must fit plaintext and tag"); assert_eq!(ours_chacha, aws_chacha); let mut opened = vec![0u8; DATA.len()]; - chacha.decrypt(&nonce, AAD, &ours_chacha, &mut opened).unwrap(); + chacha + .decrypt(&nonce, AAD, &ours_chacha, &mut opened) + .expect("freshly sealed rscrypto ChaCha20-Poly1305 ciphertext must authenticate"); assert_eq!(opened, DATA); } @@ -114,7 +125,8 @@ fn test_aws_lc_rs_ed25519_and_x25519_migration_examples_are_byte_equivalent() { use aws_lc_rs::signature::KeyPair as _; let seed = [0x13; 32]; - let aws_ed25519 = aws_signature::Ed25519KeyPair::from_seed_unchecked(&seed).unwrap(); + let aws_ed25519 = + aws_signature::Ed25519KeyPair::from_seed_unchecked(&seed).expect("32-byte AWS-LC Ed25519 seed must be valid"); let ours_ed25519 = Ed25519SecretKey::from_bytes(seed); let ours_public = ours_ed25519.public_key(); let ours_signature = ours_ed25519.sign(DATA); @@ -124,25 +136,36 @@ fn test_aws_lc_rs_ed25519_and_x25519_migration_examples_are_byte_equivalent() { aws_signature::UnparsedPublicKey::new(&aws_signature::ED25519, ours_public.as_bytes()) .verify(DATA, ours_signature.as_bytes()) - .unwrap(); - ours_public.verify(DATA, &ours_signature).unwrap(); + .expect("AWS-LC must verify the rscrypto Ed25519 signature"); + ours_public + .verify(DATA, &ours_signature) + .expect("rscrypto must verify its Ed25519 signature"); let alice_bytes = [0x18; 32]; let bob_bytes = [0x34; 32]; let ours_alice = X25519SecretKey::from_bytes(alice_bytes); let ours_bob_public = X25519SecretKey::from_bytes(bob_bytes).public_key(); - let ours_shared = ours_alice.diffie_hellman(&ours_bob_public).unwrap(); - - let aws_alice = aws_agreement::PrivateKey::from_private_key(&aws_agreement::X25519, &alice_bytes).unwrap(); - let aws_bob = aws_agreement::PrivateKey::from_private_key(&aws_agreement::X25519, &bob_bytes).unwrap(); - let aws_bob_public = aws_bob.compute_public_key().unwrap(); + let ours_shared = ours_alice + .diffie_hellman(&ours_bob_public) + .expect("fixed X25519 keys must produce a nonzero shared secret"); + + let aws_alice = aws_agreement::PrivateKey::from_private_key(&aws_agreement::X25519, &alice_bytes) + .expect("32-byte AWS-LC Alice X25519 key must be valid"); + let aws_bob = aws_agreement::PrivateKey::from_private_key(&aws_agreement::X25519, &bob_bytes) + .expect("32-byte AWS-LC Bob X25519 key must be valid"); + let aws_bob_public = aws_bob + .compute_public_key() + .expect("AWS-LC must derive Bob's X25519 public key"); let mut aws_bob_public_bytes = [0u8; 32]; aws_bob_public_bytes.copy_from_slice(aws_bob_public.as_ref()); assert_eq!(ours_bob_public.as_bytes(), &aws_bob_public_bytes); let peer = X25519PublicKey::from_bytes(aws_bob_public_bytes); assert_eq!( - ours_alice.diffie_hellman(&peer).unwrap().as_bytes(), + ours_alice + .diffie_hellman(&peer) + .expect("AWS-LC X25519 public key must produce a nonzero rscrypto shared secret") + .as_bytes(), ours_shared.as_bytes() ); @@ -152,30 +175,32 @@ fn test_aws_lc_rs_ed25519_and_x25519_migration_examples_are_byte_equivalent() { out.copy_from_slice(bytes); Ok::<[u8; 32], ()>(out) }) - .unwrap(); + .expect("AWS-LC X25519 agreement must accept the fixed key pair"); assert_eq!(ours_shared.as_bytes(), &aws_shared); } #[test] fn test_aws_lc_rs_rsa_verify_migration_examples_accept_the_same_fixtures() { - let ours = RsaPublicKey::from_spki_der(RSA3072_SPKI).unwrap(); + let ours = RsaPublicKey::from_spki_der(RSA3072_SPKI).expect("embedded RSA-3072 SPKI fixture must parse"); ours .verify_pss(RsaPssProfile::Sha256, MESSAGE_PSS, RSA3072_PSS_SHA256) - .unwrap(); + .expect("rscrypto must verify the RSA-PSS fixture"); ours .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, MESSAGE_PKCS1V15, RSA3072_PKCS1V15_SHA256) - .unwrap(); + .expect("rscrypto must verify the RSA-PKCS1-v1_5 fixture"); aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA256, RSA3072_SPKI) .verify(MESSAGE_PSS, RSA3072_PSS_SHA256) - .unwrap(); + .expect("AWS-LC must verify the RSA-PSS fixture"); aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA256, RSA3072_SPKI) .verify(MESSAGE_PKCS1V15, RSA3072_PKCS1V15_SHA256) - .unwrap(); + .expect("AWS-LC must verify the RSA-PKCS1-v1_5 fixture"); } fn aws_aead_seal(algorithm: &'static aws_aead::Algorithm, key_bytes: &[u8], plaintext: &[u8]) -> Vec { - let key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(algorithm, key_bytes).unwrap()); + let key = aws_aead::LessSafeKey::new( + aws_aead::UnboundKey::new(algorithm, key_bytes).expect("AWS-LC AEAD key length must match the algorithm"), + ); let mut out = plaintext.to_vec(); key .seal_in_place_append_tag( @@ -183,6 +208,6 @@ fn aws_aead_seal(algorithm: &'static aws_aead::Algorithm, key_bytes: &[u8], plai aws_aead::Aad::from(AAD), &mut out, ) - .unwrap(); + .expect("AWS-LC AEAD sealing must accept the fixed nonce and buffer"); out } diff --git a/tests/migration_docs.rs b/tests/migration_docs.rs index ca5fd13b..cb360982 100644 --- a/tests/migration_docs.rs +++ b/tests/migration_docs.rs @@ -59,7 +59,7 @@ fn migration_docs_do_not_delegate_accuracy_to_a_validation_index() { #[test] fn migration_docs_do_not_reference_retired_rscrypto_versions() { for path in migration_markdown_files(&migration_root()) { - let text = fs::read_to_string(&path).unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + let text = fs::read_to_string(&path).expect("migration markdown file must be readable"); assert!( !text.contains("rscrypto = { version = \"0.1\""), "{} still contains an rscrypto 0.1 dependency example", @@ -81,7 +81,7 @@ fn migration_docs_do_not_reference_retired_rscrypto_versions() { #[test] fn migration_docs_local_markdown_links_resolve() { for path in migration_markdown_files(&migration_root()) { - let text = fs::read_to_string(&path).unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + let text = fs::read_to_string(&path).expect("migration markdown file must be readable"); for link in local_markdown_links(&text) { let target = link.split_once('#').map_or(link.as_str(), |(base, _)| base); if target.is_empty() { @@ -300,8 +300,8 @@ fn migration_markdown_files(root: &Path) -> Vec { } fn collect_markdown_files(dir: &Path, out: &mut Vec) { - for entry in fs::read_dir(dir).unwrap_or_else(|err| panic!("read dir {}: {err}", dir.display())) { - let entry = entry.unwrap_or_else(|err| panic!("read dir entry in {}: {err}", dir.display())); + for entry in fs::read_dir(dir).expect("migration directory must be readable") { + let entry = entry.expect("migration directory entry must be readable"); let path = entry.path(); if path.is_dir() { collect_markdown_files(&path, out); @@ -315,14 +315,11 @@ fn local_markdown_links(text: &str) -> Vec { let mut links = Vec::new(); let mut rest = text; - while let Some(start) = rest.find("](") { - rest = &rest[start + 2..]; - let Some(end) = rest.find(')') else { + while let Some((_, after_open)) = rest.split_once("](") { + let Some((target, after_close)) = after_open.split_once(')') else { break; }; - - let target = &rest[..end]; - rest = &rest[end + 1..]; + rest = after_close; if target.starts_with("http://") || target.starts_with("https://") diff --git a/tests/migration_dryoc.rs b/tests/migration_dryoc.rs index 02332061..aa3558ea 100644 --- a/tests/migration_dryoc.rs +++ b/tests/migration_dryoc.rs @@ -14,24 +14,32 @@ const KEY_64: [u8; 64] = [0x24; 64]; #[test] fn test_dryoc_blake2b_migration_examples_are_byte_equivalent() { let mut dryoc_b256 = [0u8; 32]; - crypto_generichash(&mut dryoc_b256, DATA, None).unwrap(); + crypto_generichash(&mut dryoc_b256, DATA, None).expect("dryoc BLAKE2b-256 migration hash must succeed"); assert_eq!(Blake2b256::digest(DATA), dryoc_b256); let mut dryoc_b512 = [0u8; 64]; - crypto_generichash(&mut dryoc_b512, DATA, None).unwrap(); + crypto_generichash(&mut dryoc_b512, DATA, None).expect("dryoc BLAKE2b-512 migration hash must succeed"); assert_eq!(Blake2b512::digest(DATA), dryoc_b512); let mut dryoc_keyed_b256 = [0u8; 32]; - crypto_generichash(&mut dryoc_keyed_b256, DATA, Some(&KEY_32)).unwrap(); + crypto_generichash(&mut dryoc_keyed_b256, DATA, Some(&KEY_32)) + .expect("dryoc keyed BLAKE2b-256 migration hash must succeed"); assert_eq!( - Blake2b256::keyed_digest(Blake2bKey::new(&KEY_32).unwrap(), DATA), + Blake2b256::keyed_digest( + Blake2bKey::new(&KEY_32).expect("32-byte BLAKE2b migration key must be valid"), + DATA, + ), dryoc_keyed_b256 ); let mut dryoc_keyed_b512 = [0u8; 64]; - crypto_generichash(&mut dryoc_keyed_b512, DATA, Some(&KEY_64)).unwrap(); + crypto_generichash(&mut dryoc_keyed_b512, DATA, Some(&KEY_64)) + .expect("dryoc keyed BLAKE2b-512 migration hash must succeed"); assert_eq!( - Blake2b512::keyed_digest(Blake2bKey::new(&KEY_64).unwrap(), DATA), + Blake2b512::keyed_digest( + Blake2bKey::new(&KEY_64).expect("64-byte BLAKE2b migration key must be valid"), + DATA, + ), dryoc_keyed_b512 ); } @@ -41,7 +49,8 @@ fn test_dryoc_ed25519_migration_examples_are_byte_equivalent() { let seed = [0x13; 32]; let (dryoc_public, dryoc_secret) = crypto_sign_seed_keypair(&seed); let mut dryoc_signature = [0u8; 64]; - crypto_sign_detached(&mut dryoc_signature, DATA, &dryoc_secret).unwrap(); + crypto_sign_detached(&mut dryoc_signature, DATA, &dryoc_secret) + .expect("dryoc Ed25519 migration signing must succeed"); let ours = Ed25519SecretKey::from_bytes(seed); let ours_public = ours.public_key(); @@ -50,8 +59,11 @@ fn test_dryoc_ed25519_migration_examples_are_byte_equivalent() { assert_eq!(ours_public.as_bytes(), &dryoc_public); assert_eq!(ours_signature.as_bytes(), &dryoc_signature); - crypto_sign_verify_detached(ours_signature.as_bytes(), DATA, &dryoc_public).unwrap(); - ours_public.verify(DATA, &ours_signature).unwrap(); + crypto_sign_verify_detached(ours_signature.as_bytes(), DATA, &dryoc_public) + .expect("dryoc must verify the rscrypto Ed25519 migration signature"); + ours_public + .verify(DATA, &ours_signature) + .expect("rscrypto must verify its Ed25519 migration signature"); } #[test] @@ -61,13 +73,16 @@ fn test_dryoc_x25519_migration_examples_are_byte_equivalent() { let ours_alice = X25519SecretKey::from_bytes(alice_bytes); let ours_bob_public = X25519SecretKey::from_bytes(bob_bytes).public_key(); - let ours_shared = ours_alice.diffie_hellman(&ours_bob_public).unwrap(); + let ours_shared = ours_alice + .diffie_hellman(&ours_bob_public) + .expect("rscrypto X25519 migration exchange must produce a nonzero secret"); let mut dryoc_bob_public = [0u8; 32]; crypto_scalarmult_base(&mut dryoc_bob_public, &bob_bytes); assert_eq!(ours_bob_public.as_bytes(), &dryoc_bob_public); let mut dryoc_shared = [0u8; 32]; - crypto_scalarmult(&mut dryoc_shared, &alice_bytes, &dryoc_bob_public); + crypto_scalarmult(&mut dryoc_shared, &alice_bytes, &dryoc_bob_public) + .expect("dryoc X25519 migration exchange must produce a nonzero secret"); assert_eq!(ours_shared.as_bytes(), &dryoc_shared); } diff --git a/tests/migration_ring.rs b/tests/migration_ring.rs index a2b51d44..70edbb04 100644 --- a/tests/migration_ring.rs +++ b/tests/migration_ring.rs @@ -53,15 +53,17 @@ fn test_ring_digest_hmac_hkdf_and_pbkdf2_migration_examples_are_byte_equivalent( ring_hkdf::Salt::new(ring_hkdf::HKDF_SHA256, salt) .extract(ikm) .expand(&[info], RingHkdfLen(ring_okm.len())) - .unwrap() + .expect("ring HKDF must accept the migration output length") .fill(&mut ring_okm) - .unwrap(); + .expect("ring HKDF migration expansion must succeed"); let mut ours_okm = [0u8; 42]; - HkdfSha256::new(salt, ikm).expand(info, &mut ours_okm).unwrap(); + HkdfSha256::new(salt, ikm) + .expand(info, &mut ours_okm) + .expect("rscrypto HKDF migration expansion must succeed"); assert_eq!(ours_okm, ring_okm); - let iterations = NonZeroU32::new(600_000).unwrap(); + let iterations = NonZeroU32::new(600_000).expect("migration iteration count must be nonzero"); let mut ring_pbkdf2 = [0u8; 32]; ring_pbkdf2::derive( ring_pbkdf2::PBKDF2_HMAC_SHA256, @@ -72,7 +74,8 @@ fn test_ring_digest_hmac_hkdf_and_pbkdf2_migration_examples_are_byte_equivalent( ); let mut ours_pbkdf2 = [0u8; 32]; - Pbkdf2Sha256::derive_key(b"ring migration password", salt, iterations.get(), &mut ours_pbkdf2).unwrap(); + Pbkdf2Sha256::derive_key(b"ring migration password", salt, iterations.get(), &mut ours_pbkdf2) + .expect("rscrypto PBKDF2 migration derivation must succeed"); assert_eq!(ours_pbkdf2, ring_pbkdf2); } @@ -82,13 +85,17 @@ fn test_ring_aead_migration_examples_are_byte_equivalent() { let aes = Aes256Gcm::new(&Aes256GcmKey::from_bytes(KEY_32)); let nonce = Nonce96::from_bytes(NONCE_12); let mut ours_aes = vec![0u8; DATA.len() + 16]; - aes.encrypt(&nonce, AAD, DATA, &mut ours_aes).unwrap(); + aes + .encrypt(&nonce, AAD, DATA, &mut ours_aes) + .expect("rscrypto AES-GCM migration encryption must succeed"); assert_eq!(ours_aes, ring_aes); let ring_chacha = ring_aead_seal(&ring_aead::CHACHA20_POLY1305, &KEY_32, DATA); let chacha = ChaCha20Poly1305::new(&ChaCha20Poly1305Key::from_bytes(KEY_32)); let mut ours_chacha = vec![0u8; DATA.len() + 16]; - chacha.encrypt(&nonce, AAD, DATA, &mut ours_chacha).unwrap(); + chacha + .encrypt(&nonce, AAD, DATA, &mut ours_chacha) + .expect("rscrypto ChaCha20-Poly1305 migration encryption must succeed"); assert_eq!(ours_chacha, ring_chacha); } @@ -97,7 +104,8 @@ fn test_ring_ed25519_and_rsa_verify_migration_examples_are_compatible() { use ring::signature::KeyPair as _; let seed = [0x13; 32]; - let ring_ed25519 = ring::signature::Ed25519KeyPair::from_seed_unchecked(&seed).unwrap(); + let ring_ed25519 = + ring::signature::Ed25519KeyPair::from_seed_unchecked(&seed).expect("ring must accept the Ed25519 migration seed"); let ours_ed25519 = Ed25519SecretKey::from_bytes(seed); let ours_public = ours_ed25519.public_key(); let ours_signature = ours_ed25519.sign(DATA); @@ -107,28 +115,32 @@ fn test_ring_ed25519_and_rsa_verify_migration_examples_are_compatible() { ring::signature::UnparsedPublicKey::new(&ring::signature::ED25519, ours_public.as_bytes()) .verify(DATA, ours_signature.as_bytes()) - .unwrap(); - ours_public.verify(DATA, &ours_signature).unwrap(); + .expect("ring must verify the rscrypto Ed25519 migration signature"); + ours_public + .verify(DATA, &ours_signature) + .expect("rscrypto must verify its Ed25519 migration signature"); - let ours = RsaPublicKey::from_spki_der(RSA3072_SPKI).unwrap(); + let ours = RsaPublicKey::from_spki_der(RSA3072_SPKI).expect("RSA migration SPKI fixture must parse"); let pkcs1 = ours.to_pkcs1_der(); ours .verify_pss(RsaPssProfile::Sha256, MESSAGE_PSS, RSA3072_PSS_SHA256) - .unwrap(); + .expect("rscrypto must verify the RSA-PSS migration fixture"); ours .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, MESSAGE_PKCS1V15, RSA3072_PKCS1V15_SHA256) - .unwrap(); + .expect("rscrypto must verify the RSA-PKCS1-v1_5 migration fixture"); ring::signature::UnparsedPublicKey::new(&ring::signature::RSA_PSS_2048_8192_SHA256, &pkcs1) .verify(MESSAGE_PSS, RSA3072_PSS_SHA256) - .unwrap(); + .expect("ring must verify the RSA-PSS migration fixture"); ring::signature::UnparsedPublicKey::new(&ring::signature::RSA_PKCS1_2048_8192_SHA256, &pkcs1) .verify(MESSAGE_PKCS1V15, RSA3072_PKCS1V15_SHA256) - .unwrap(); + .expect("ring must verify the RSA-PKCS1-v1_5 migration fixture"); } fn ring_aead_seal(algorithm: &'static ring_aead::Algorithm, key_bytes: &[u8], plaintext: &[u8]) -> Vec { - let key = ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(algorithm, key_bytes).unwrap()); + let unbound_key = + ring_aead::UnboundKey::new(algorithm, key_bytes).expect("ring must accept the migration AEAD key length"); + let key = ring_aead::LessSafeKey::new(unbound_key); let mut out = plaintext.to_vec(); key .seal_in_place_append_tag( @@ -136,6 +148,6 @@ fn ring_aead_seal(algorithm: &'static ring_aead::Algorithm, key_bytes: &[u8], pl ring_aead::Aad::from(AAD), &mut out, ) - .unwrap(); + .expect("ring migration AEAD encryption must succeed"); out } diff --git a/tests/mlkem_acvp.rs b/tests/mlkem_acvp.rs index 9d7686c2..de0d781e 100644 --- a/tests/mlkem_acvp.rs +++ b/tests/mlkem_acvp.rs @@ -15,28 +15,44 @@ const ACVP_ENCAP_DECAP_PROMPT: &str = include_str!("vectors/mlkem_acvp_encapdeca const ACVP_ENCAP_DECAP_EXPECTED: &str = include_str!("vectors/mlkem_acvp_encapdecap_fips203_expected.json"); fn parse_acvp(json: &str) -> Value { - serde_json::from_str(json).unwrap() + serde_json::from_str(json).expect("embedded ACVP fixture must contain valid JSON") } fn string_field<'a>(value: &'a Value, field: &str) -> &'a str { value .get(field) .and_then(Value::as_str) - .unwrap_or_else(|| panic!("missing ACVP string field `{field}`")) + .expect("ACVP string field must exist and contain a string") } fn bool_field(value: &Value, field: &str) -> bool { value .get(field) .and_then(Value::as_bool) - .unwrap_or_else(|| panic!("missing ACVP bool field `{field}`")) + .expect("ACVP bool field must exist and contain a boolean") } fn u64_field(value: &Value, field: &str) -> u64 { value .get(field) .and_then(Value::as_u64) - .unwrap_or_else(|| panic!("missing ACVP integer field `{field}`")) + .expect("ACVP integer field must exist and contain an unsigned integer") +} + +enum ParameterSet { + MlKem512, + MlKem768, + MlKem1024, +} + +fn parameter_set(group: &Value) -> ParameterSet { + let parameter_set = match string_field(group, "parameterSet") { + "ML-KEM-512" => Some(ParameterSet::MlKem512), + "ML-KEM-768" => Some(ParameterSet::MlKem768), + "ML-KEM-1024" => Some(ParameterSet::MlKem1024), + _ => None, + }; + parameter_set.expect("ACVP parameterSet must name a supported ML-KEM profile") } fn test_groups(vectors: &Value) -> &[Value] { @@ -59,22 +75,22 @@ fn group_by_id(vectors: &Value, tg_id: u64) -> &Value { test_groups(vectors) .iter() .find(|group| u64_field(group, "tgId") == tg_id) - .unwrap_or_else(|| panic!("missing ACVP expected group tgId {tg_id}")) + .expect("ACVP expected group must contain every prompt tgId") } fn case_by_id(group: &Value, tc_id: u64) -> &Value { test_cases(group) .iter() .find(|test_case| u64_field(test_case, "tcId") == tc_id) - .unwrap_or_else(|| panic!("missing ACVP expected tcId {tc_id}")) + .expect("ACVP expected group must contain every prompt tcId") } -fn hex_nibble(byte: u8) -> u8 { +fn hex_nibble(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex byte {byte:#x}"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } @@ -83,7 +99,9 @@ fn decode_hex(hex: &str) -> Vec { let mut out = Vec::with_capacity(hex.len() / 2); for pair in hex.as_bytes().chunks_exact(2) { - out.push((hex_nibble(pair[0]) << 4) | hex_nibble(pair[1])); + let high = hex_nibble(pair[0]).expect("ACVP fixture must contain only hexadecimal digits"); + let low = hex_nibble(pair[1]).expect("ACVP fixture must contain only hexadecimal digits"); + out.push((high << 4) | low); } out } @@ -117,7 +135,7 @@ macro_rules! assert_keygen_case { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("ML-KEM ACVP key generation must succeed"); let expected_encapsulation_key = array_from_hex::<{ <$profile>::ENCAPSULATION_KEY_SIZE }>(string_field($expected, "ek")); @@ -145,14 +163,15 @@ macro_rules! assert_encapsulation_case { ($profile:ty, $encapsulation_key:ty, $parameter_set:literal, $prompt:expr, $expected:expr) => {{ let tc_id = u64_field($prompt, "tcId"); let encapsulation_key_bytes = decode_hex(string_field($prompt, "ek")); - let encapsulation_key = <$encapsulation_key>::try_from_slice(&encapsulation_key_bytes).unwrap(); + let encapsulation_key = <$encapsulation_key>::try_from_slice(&encapsulation_key_bytes) + .expect("ACVP encapsulation key must have a valid encoding"); let random = array_from_hex::<{ <$profile>::ENCAPSULATION_RANDOM_SIZE }>(string_field($prompt, "m")); let (ciphertext, shared_secret) = <$profile>::encapsulate(&encapsulation_key, |out| { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("ML-KEM ACVP encapsulation must succeed"); let expected_ciphertext = array_from_hex::<{ <$profile>::CIPHERTEXT_SIZE }>(string_field($expected, "c")); let expected_shared_secret = array_from_hex::<{ <$profile>::SHARED_SECRET_SIZE }>(string_field($expected, "k")); @@ -173,9 +192,12 @@ macro_rules! assert_decapsulation_case { let tc_id = u64_field($prompt, "tcId"); let decapsulation_key_bytes = decode_hex(string_field($prompt, "dk")); let ciphertext_bytes = decode_hex(string_field($prompt, "c")); - let decapsulation_key = <$decapsulation_key>::try_from_slice(&decapsulation_key_bytes).unwrap(); - let ciphertext = <$ciphertext>::try_from_slice(&ciphertext_bytes).unwrap(); - let shared_secret = <$profile>::decapsulate(&decapsulation_key, &ciphertext).unwrap(); + let decapsulation_key = <$decapsulation_key>::try_from_slice(&decapsulation_key_bytes) + .expect("ACVP decapsulation key must have a valid encoding"); + let ciphertext = + <$ciphertext>::try_from_slice(&ciphertext_bytes).expect("ACVP ciphertext must have a valid encoding"); + let shared_secret = + <$profile>::decapsulate(&decapsulation_key, &ciphertext).expect("ML-KEM ACVP decapsulation must succeed"); let expected_shared_secret = array_from_hex::<{ <$profile>::SHARED_SECRET_SIZE }>(string_field($expected, "k")); assert_bytes_eq( @@ -215,11 +237,10 @@ fn mlkem_matches_acvp_keygen_fips203_vectors() { for prompt_case in test_cases(prompt_group) { let expected_case = case_by_id(expected_group, u64_field(prompt_case, "tcId")); - match string_field(prompt_group, "parameterSet") { - "ML-KEM-512" => assert_keygen_case!(MlKem512, "ML-KEM-512", prompt_case, expected_case), - "ML-KEM-768" => assert_keygen_case!(MlKem768, "ML-KEM-768", prompt_case, expected_case), - "ML-KEM-1024" => assert_keygen_case!(MlKem1024, "ML-KEM-1024", prompt_case, expected_case), - parameter_set => panic!("unsupported ACVP ML-KEM parameter set `{parameter_set}`"), + match parameter_set(prompt_group) { + ParameterSet::MlKem512 => assert_keygen_case!(MlKem512, "ML-KEM-512", prompt_case, expected_case), + ParameterSet::MlKem768 => assert_keygen_case!(MlKem768, "ML-KEM-768", prompt_case, expected_case), + ParameterSet::MlKem1024 => assert_keygen_case!(MlKem1024, "ML-KEM-1024", prompt_case, expected_case), } } } @@ -240,8 +261,8 @@ fn mlkem_matches_acvp_encapsulation_fips203_vectors() { for prompt_case in test_cases(prompt_group) { let expected_case = case_by_id(expected_group, u64_field(prompt_case, "tcId")); - match string_field(prompt_group, "parameterSet") { - "ML-KEM-512" => { + match parameter_set(prompt_group) { + ParameterSet::MlKem512 => { assert_encapsulation_case!( MlKem512, MlKem512EncapsulationKey, @@ -250,7 +271,7 @@ fn mlkem_matches_acvp_encapsulation_fips203_vectors() { expected_case ) } - "ML-KEM-768" => { + ParameterSet::MlKem768 => { assert_encapsulation_case!( MlKem768, MlKem768EncapsulationKey, @@ -259,7 +280,7 @@ fn mlkem_matches_acvp_encapsulation_fips203_vectors() { expected_case ) } - "ML-KEM-1024" => { + ParameterSet::MlKem1024 => { assert_encapsulation_case!( MlKem1024, MlKem1024EncapsulationKey, @@ -268,7 +289,6 @@ fn mlkem_matches_acvp_encapsulation_fips203_vectors() { expected_case ) } - parameter_set => panic!("unsupported ACVP ML-KEM parameter set `{parameter_set}`"), } } } @@ -289,8 +309,8 @@ fn mlkem_matches_acvp_decapsulation_fips203_vectors() { for prompt_case in test_cases(prompt_group) { let expected_case = case_by_id(expected_group, u64_field(prompt_case, "tcId")); - match string_field(prompt_group, "parameterSet") { - "ML-KEM-512" => assert_decapsulation_case!( + match parameter_set(prompt_group) { + ParameterSet::MlKem512 => assert_decapsulation_case!( MlKem512, MlKem512DecapsulationKey, MlKem512Ciphertext, @@ -298,7 +318,7 @@ fn mlkem_matches_acvp_decapsulation_fips203_vectors() { prompt_case, expected_case ), - "ML-KEM-768" => assert_decapsulation_case!( + ParameterSet::MlKem768 => assert_decapsulation_case!( MlKem768, MlKem768DecapsulationKey, MlKem768Ciphertext, @@ -306,7 +326,7 @@ fn mlkem_matches_acvp_decapsulation_fips203_vectors() { prompt_case, expected_case ), - "ML-KEM-1024" => assert_decapsulation_case!( + ParameterSet::MlKem1024 => assert_decapsulation_case!( MlKem1024, MlKem1024DecapsulationKey, MlKem1024Ciphertext, @@ -314,7 +334,6 @@ fn mlkem_matches_acvp_decapsulation_fips203_vectors() { prompt_case, expected_case ), - parameter_set => panic!("unsupported ACVP ML-KEM parameter set `{parameter_set}`"), } } } @@ -335,14 +354,14 @@ fn mlkem_matches_acvp_decapsulation_key_check_fips203_vectors() { for prompt_case in test_cases(prompt_group) { let expected_case = case_by_id(expected_group, u64_field(prompt_case, "tcId")); - match string_field(prompt_group, "parameterSet") { - "ML-KEM-512" => { + match parameter_set(prompt_group) { + ParameterSet::MlKem512 => { assert_key_check_case!(MlKem512DecapsulationKey, "ML-KEM-512", prompt_case, expected_case, "dk") } - "ML-KEM-768" => { + ParameterSet::MlKem768 => { assert_key_check_case!(MlKem768DecapsulationKey, "ML-KEM-768", prompt_case, expected_case, "dk") } - "ML-KEM-1024" => { + ParameterSet::MlKem1024 => { assert_key_check_case!( MlKem1024DecapsulationKey, "ML-KEM-1024", @@ -351,7 +370,6 @@ fn mlkem_matches_acvp_decapsulation_key_check_fips203_vectors() { "dk" ) } - parameter_set => panic!("unsupported ACVP ML-KEM parameter set `{parameter_set}`"), } } } @@ -372,14 +390,14 @@ fn mlkem_matches_acvp_encapsulation_key_check_fips203_vectors() { for prompt_case in test_cases(prompt_group) { let expected_case = case_by_id(expected_group, u64_field(prompt_case, "tcId")); - match string_field(prompt_group, "parameterSet") { - "ML-KEM-512" => { + match parameter_set(prompt_group) { + ParameterSet::MlKem512 => { assert_key_check_case!(MlKem512EncapsulationKey, "ML-KEM-512", prompt_case, expected_case, "ek") } - "ML-KEM-768" => { + ParameterSet::MlKem768 => { assert_key_check_case!(MlKem768EncapsulationKey, "ML-KEM-768", prompt_case, expected_case, "ek") } - "ML-KEM-1024" => { + ParameterSet::MlKem1024 => { assert_key_check_case!( MlKem1024EncapsulationKey, "ML-KEM-1024", @@ -388,7 +406,6 @@ fn mlkem_matches_acvp_encapsulation_key_check_fips203_vectors() { "ek" ) } - parameter_set => panic!("unsupported ACVP ML-KEM parameter set `{parameter_set}`"), } } } diff --git a/tests/mlkem_ops.rs b/tests/mlkem_ops.rs index 3b0adcf3..cc70ea6f 100644 --- a/tests/mlkem_ops.rs +++ b/tests/mlkem_ops.rs @@ -11,7 +11,8 @@ use rscrypto::{ fn deterministic_bytes(offset: u8) -> [u8; N] { let mut out = [0u8; N]; for (i, byte) in out.iter_mut().enumerate() { - *byte = offset.wrapping_add(i as u8); + let i = u8::try_from(i).expect("ML-KEM deterministic entropy index must fit in u8"); + *byte = offset.wrapping_add(i); } out } diff --git a/tests/mlkem_properties.rs b/tests/mlkem_properties.rs index 9c3d1f1d..5f0e34c4 100644 --- a/tests/mlkem_properties.rs +++ b/tests/mlkem_properties.rs @@ -44,7 +44,7 @@ fn mlkem512_matches_fips203_for_reduced_feature_ci_seed() { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("deterministic ML-KEM-512 key generation must succeed"); let (fips_ek, fips_dk) = fips_mlkem512::KG::keygen_from_seed(d, z); assert_eq!(*ek.as_bytes(), fips_ek.clone().into_bytes()); @@ -54,7 +54,7 @@ fn mlkem512_matches_fips203_for_reduced_feature_ci_seed() { out.copy_from_slice(&m); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("deterministic ML-KEM-512 encapsulation must succeed"); let (fips_shared_secret, fips_ciphertext) = fips_ek.encaps_from_seed(&m); assert_eq!(*ciphertext.as_bytes(), fips_ciphertext.clone().into_bytes()); @@ -63,8 +63,10 @@ fn mlkem512_matches_fips203_for_reduced_feature_ci_seed() { fips_shared_secret.into_bytes() ); - let decapsulated = MlKem512::decapsulate(&dk, &ciphertext).unwrap(); - let fips_decapsulated = fips_dk.try_decaps(&fips_ciphertext).unwrap(); + let decapsulated = MlKem512::decapsulate(&dk, &ciphertext).expect("generated ML-KEM-512 ciphertext must decapsulate"); + let fips_decapsulated = fips_dk + .try_decaps(&fips_ciphertext) + .expect("FIPS 203 ML-KEM-512 ciphertext must decapsulate"); assert_eq!(*decapsulated.expose_secret().as_bytes(), fips_decapsulated.into_bytes()); } @@ -94,7 +96,7 @@ macro_rules! mlkem_profile_properties { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("deterministic ML-KEM key generation must succeed"); let (fips_ek, fips_dk) = $fips::KG::keygen_from_seed(d, z); prop_assert_eq!(*ek.as_bytes(), fips_ek.clone().into_bytes()); @@ -104,14 +106,17 @@ macro_rules! mlkem_profile_properties { out.copy_from_slice(&m); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("deterministic ML-KEM encapsulation must succeed"); let (fips_shared_secret, fips_ciphertext) = fips_ek.encaps_from_seed(&m); prop_assert_eq!(*ciphertext.as_bytes(), fips_ciphertext.clone().into_bytes()); prop_assert_eq!(*shared_secret.expose_secret().as_bytes(), fips_shared_secret.into_bytes()); - let decapsulated = <$profile>::decapsulate(&dk, &ciphertext).unwrap(); - let fips_decapsulated = fips_dk.try_decaps(&fips_ciphertext).unwrap(); + let decapsulated = <$profile>::decapsulate(&dk, &ciphertext) + .expect("generated ML-KEM ciphertext must decapsulate"); + let fips_decapsulated = fips_dk + .try_decaps(&fips_ciphertext) + .expect("FIPS 203 ML-KEM ciphertext must decapsulate"); prop_assert_eq!(*decapsulated.expose_secret().as_bytes(), fips_decapsulated.into_bytes()); } @@ -128,16 +133,17 @@ macro_rules! mlkem_profile_properties { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("deterministic ML-KEM key generation must succeed"); let (ciphertext, encapsulated) = <$profile>::encapsulate(&ek, |out| { out.copy_from_slice(&m); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("deterministic ML-KEM encapsulation must succeed"); let mut modified = ciphertext.to_bytes(); modified[byte_idx] ^= 1u8 << bit_idx; - let rejected = <$profile>::decapsulate(&dk, &<$ciphertext>::from_bytes(modified)).unwrap(); + let rejected = <$profile>::decapsulate(&dk, &<$ciphertext>::from_bytes(modified)) + .expect("modified ML-KEM ciphertext must use implicit rejection"); prop_assert!( !encapsulated.ct_eq(&rejected).declassify(), @@ -150,26 +156,68 @@ macro_rules! mlkem_profile_properties { fn $slice_parsers(bytes in prop::collection::vec(any::(), 0..=3400)) { if bytes.len() != <$profile>::ENCAPSULATION_KEY_SIZE { prop_assert_eq!( - <$encapsulation_key>::try_from_slice(&bytes).unwrap_err(), + <$encapsulation_key>::try_from_slice(&bytes) + .expect_err("wrong-length ML-KEM encapsulation key must be rejected"), MlKemError::InvalidEncapsulationKey ); } else { - let _ = <$encapsulation_key>::try_from_slice(&bytes); + let mut raw = [0u8; <$profile>::ENCAPSULATION_KEY_SIZE]; + raw.copy_from_slice(&bytes); + let direct = <$encapsulation_key>::from_bytes(raw); + match direct.validate() { + Ok(()) => { + let parsed = <$encapsulation_key>::try_from_slice(&bytes) + .expect("canonical ML-KEM encapsulation key must parse"); + prop_assert_eq!(parsed.as_ref(), bytes.as_slice()); + } + Err(expected) => { + prop_assert_eq!( + <$encapsulation_key>::try_from_slice(&bytes) + .expect_err("noncanonical ML-KEM encapsulation key must be rejected"), + expected + ); + } + } } if bytes.len() != <$profile>::DECAPSULATION_KEY_SIZE { prop_assert_eq!( - <$decapsulation_key>::try_from_slice(&bytes).unwrap_err(), + <$decapsulation_key>::try_from_slice(&bytes) + .expect_err("wrong-length ML-KEM decapsulation key must be rejected"), MlKemError::InvalidDecapsulationKey ); } else { - let _ = <$decapsulation_key>::try_from_slice(&bytes); + let mut raw = [0u8; <$profile>::DECAPSULATION_KEY_SIZE]; + raw.copy_from_slice(&bytes); + let direct = <$decapsulation_key>::from_bytes(raw); + match direct.validate() { + Ok(()) => { + let parsed = <$decapsulation_key>::try_from_slice(&bytes) + .expect("valid ML-KEM decapsulation key must parse"); + prop_assert_eq!(parsed.as_ref(), bytes.as_slice()); + } + Err(expected) => { + prop_assert_eq!( + <$decapsulation_key>::try_from_slice(&bytes) + .expect_err("invalid ML-KEM decapsulation key must be rejected"), + expected + ); + } + } } if bytes.len() != <$profile>::CIPHERTEXT_SIZE { - prop_assert_eq!(<$ciphertext>::try_from_slice(&bytes).unwrap_err(), MlKemError::InvalidCiphertext); + prop_assert_eq!( + <$ciphertext>::try_from_slice(&bytes) + .expect_err("wrong-length ML-KEM ciphertext must be rejected"), + MlKemError::InvalidCiphertext + ); } else { - <$ciphertext>::try_from_slice(&bytes).unwrap().validate().unwrap(); + let ciphertext = <$ciphertext>::try_from_slice(&bytes) + .expect("correct-length ML-KEM ciphertext must parse"); + ciphertext + .validate() + .expect("all correctly sized ML-KEM ciphertexts are canonical"); } } } diff --git a/tests/owned_equality.rs b/tests/owned_equality.rs index aadfa205..1029d46d 100644 --- a/tests/owned_equality.rs +++ b/tests/owned_equality.rs @@ -1,3 +1,4 @@ +#[cfg(any(feature = "aes-gcm", feature = "hmac", feature = "x25519"))] fn assert_owned_equality(make: impl Fn([u8; N]) -> T, compare: impl Fn(&T, &T) -> bool) { let bytes = [0x5a; N]; assert!(compare(&make(bytes), &make(bytes))); diff --git a/tests/pbkdf2_differential.rs b/tests/pbkdf2_differential.rs index d2dbb3f6..eefc1bac 100644 --- a/tests/pbkdf2_differential.rs +++ b/tests/pbkdf2_differential.rs @@ -32,21 +32,27 @@ proptest! { oracle_sha256(&password, &salt, iterations, &mut expected); let mut oneshot = vec![0u8; out_len]; - Pbkdf2Sha256::derive_key_primitive(&password, &salt, iterations, &mut oneshot).unwrap(); + Pbkdf2Sha256::derive_key_primitive(&password, &salt, iterations, &mut oneshot) + .expect("PBKDF2-SHA-256 one-shot differential derivation must succeed"); let state = Pbkdf2Sha256::new(&password); let mut reused = vec![0u8; out_len]; - state.derive(&salt, iterations, &mut reused).unwrap(); + state.derive(&salt, iterations, &mut reused) + .expect("PBKDF2-SHA-256 state-reuse differential derivation must succeed"); prop_assert_eq!(oneshot, expected.as_slice()); prop_assert_eq!(reused, expected.as_slice()); - prop_assert!(state.verify_primitive(&salt, iterations, &expected).is_ok()); - prop_assert!(Pbkdf2Sha256::verify_password_primitive(&password, &salt, iterations, &expected).is_ok()); + state.verify_primitive(&salt, iterations, &expected) + .expect("PBKDF2-SHA-256 state must verify the oracle output"); + Pbkdf2Sha256::verify_password_primitive(&password, &salt, iterations, &expected) + .expect("PBKDF2-SHA-256 one-shot API must verify the oracle output"); let mut wrong = expected.clone(); wrong[0] ^= 1; - prop_assert!(state.verify_primitive(&salt, iterations, &wrong).is_err()); - prop_assert!(Pbkdf2Sha256::verify_password_primitive(&password, &salt, iterations, &wrong).is_err()); + state.verify_primitive(&salt, iterations, &wrong) + .expect_err("PBKDF2-SHA-256 state must reject a corrupted output"); + Pbkdf2Sha256::verify_password_primitive(&password, &salt, iterations, &wrong) + .expect_err("PBKDF2-SHA-256 one-shot API must reject a corrupted output"); } #[test] @@ -60,21 +66,27 @@ proptest! { oracle_sha512(&password, &salt, iterations, &mut expected); let mut oneshot = vec![0u8; out_len]; - Pbkdf2Sha512::derive_key_primitive(&password, &salt, iterations, &mut oneshot).unwrap(); + Pbkdf2Sha512::derive_key_primitive(&password, &salt, iterations, &mut oneshot) + .expect("PBKDF2-SHA-512 one-shot differential derivation must succeed"); let state = Pbkdf2Sha512::new(&password); let mut reused = vec![0u8; out_len]; - state.derive(&salt, iterations, &mut reused).unwrap(); + state.derive(&salt, iterations, &mut reused) + .expect("PBKDF2-SHA-512 state-reuse differential derivation must succeed"); prop_assert_eq!(oneshot, expected.as_slice()); prop_assert_eq!(reused, expected.as_slice()); - prop_assert!(state.verify_primitive(&salt, iterations, &expected).is_ok()); - prop_assert!(Pbkdf2Sha512::verify_password_primitive(&password, &salt, iterations, &expected).is_ok()); + state.verify_primitive(&salt, iterations, &expected) + .expect("PBKDF2-SHA-512 state must verify the oracle output"); + Pbkdf2Sha512::verify_password_primitive(&password, &salt, iterations, &expected) + .expect("PBKDF2-SHA-512 one-shot API must verify the oracle output"); let mut wrong = expected.clone(); wrong[0] ^= 1; - prop_assert!(state.verify_primitive(&salt, iterations, &wrong).is_err()); - prop_assert!(Pbkdf2Sha512::verify_password_primitive(&password, &salt, iterations, &wrong).is_err()); + state.verify_primitive(&salt, iterations, &wrong) + .expect_err("PBKDF2-SHA-512 state must reject a corrupted output"); + Pbkdf2Sha512::verify_password_primitive(&password, &salt, iterations, &wrong) + .expect_err("PBKDF2-SHA-512 one-shot API must reject a corrupted output"); } } @@ -106,7 +118,8 @@ fn pbkdf2_sha256_rfc7914_vectors_match_rustcrypto() { ]; for case in cases { - let actual = Pbkdf2Sha256::derive_key_array_primitive::<64>(case.password, case.salt, case.iterations).unwrap(); + let actual = Pbkdf2Sha256::derive_key_array_primitive::<64>(case.password, case.salt, case.iterations) + .expect("RFC 7914 PBKDF2-SHA-256 vector derivation must succeed"); assert_eq!(actual, case.expected); let mut oracle = [0u8; 64]; diff --git a/tests/pbkdf2_kat_vectors.rs b/tests/pbkdf2_kat_vectors.rs index cbc4ab1d..5e8062d3 100644 --- a/tests/pbkdf2_kat_vectors.rs +++ b/tests/pbkdf2_kat_vectors.rs @@ -17,12 +17,23 @@ use rscrypto::{Pbkdf2Sha256, Pbkdf2Sha512}; -fn hex_to_bytes(s: &str) -> Vec { - let s: String = s.chars().filter(|c| !c.is_ascii_whitespace()).collect(); - (0..s.len()) - .step_by(2) - .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) - .collect() +fn hex_to_bytes(source: &str) -> Vec { + let mut digits = source.chars().filter(|character| !character.is_ascii_whitespace()); + let mut output = Vec::new(); + + while let Some(high) = digits.next() { + let low = digits.next().expect("KAT hex input must contain complete byte pairs"); + let high = high + .to_digit(16) + .expect("KAT input must contain only hexadecimal digits"); + let low = low + .to_digit(16) + .expect("KAT input must contain only hexadecimal digits"); + let byte = high.strict_mul(16).strict_add(low); + output.push(u8::try_from(byte).expect("two hexadecimal digits must fit in one byte")); + } + + output } #[test] @@ -30,7 +41,8 @@ fn pbkdf2_sha256_kat_c1_dk32() { // P="password" S="salt" c=1 dk_len=32 let expected = hex_to_bytes("120fb6cffcf8b32c43e7225256c4f837 a86548c92ccc35480805987cb70be17b"); let mut out = [0u8; 32]; - Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 1, &mut out).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 1, &mut out) + .expect("PBKDF2-SHA-256 c=1 KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -39,7 +51,8 @@ fn pbkdf2_sha256_kat_c2_dk32() { // P="password" S="salt" c=2 dk_len=32 let expected = hex_to_bytes("ae4d0c95af6b46d32d0adff928f06dd0 2a303f8ef3c251dfd6e2d85a95474c43"); let mut out = [0u8; 32]; - Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 2, &mut out).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 2, &mut out) + .expect("PBKDF2-SHA-256 c=2 KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -49,7 +62,8 @@ fn pbkdf2_sha256_kat_c4096_dk32() { // P="password" S="salt" c=4096 dk_len=32 let expected = hex_to_bytes("c5e478d59288c841aa530db6845c4c8d 962893a001ce4e11a4963873aa98134a"); let mut out = [0u8; 32]; - Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 4096, &mut out).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 4096, &mut out) + .expect("PBKDF2-SHA-256 c=4096 KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -65,7 +79,7 @@ fn pbkdf2_sha256_kat_long_inputs_c4096_dk40() { 4096, &mut out, ) - .unwrap(); + .expect("PBKDF2-SHA-256 long-input KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -75,7 +89,8 @@ fn pbkdf2_sha256_kat_embedded_nul_c4096_dk16() { // NUL-termination bugs in the HMAC key/salt path. let expected = hex_to_bytes("89b69d0516f829893c696226650a8687"); let mut out = [0u8; 16]; - Pbkdf2Sha256::derive_key_primitive(b"pass\x00word", b"sa\x00lt", 4096, &mut out).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"pass\x00word", b"sa\x00lt", 4096, &mut out) + .expect("PBKDF2-SHA-256 embedded-NUL KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -87,7 +102,8 @@ fn pbkdf2_sha512_kat_c1_dk64() { 050235d7d68b1da55e63f73b60a57fce", ); let mut out = [0u8; 64]; - Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 1, &mut out).unwrap(); + Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 1, &mut out) + .expect("PBKDF2-SHA-512 c=1 KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -99,7 +115,8 @@ fn pbkdf2_sha512_kat_c2_dk64() { be67335c77a6068e04112754f27ccf4e", ); let mut out = [0u8; 64]; - Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 2, &mut out).unwrap(); + Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 2, &mut out) + .expect("PBKDF2-SHA-512 c=2 KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -112,7 +129,8 @@ fn pbkdf2_sha512_kat_c4096_dk64() { 376060ecd532e039b742a239434af2d5", ); let mut out = [0u8; 64]; - Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 4096, &mut out).unwrap(); + Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 4096, &mut out) + .expect("PBKDF2-SHA-512 c=4096 KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -120,19 +138,25 @@ fn pbkdf2_sha512_kat_c4096_dk64() { fn pbkdf2_sha256_state_reuse_matches_oneshot() { let state = Pbkdf2Sha256::new(b"password"); let mut from_state = [0u8; 32]; - state.derive(b"salt", 100, &mut from_state).unwrap(); + state + .derive(b"salt", 100, &mut from_state) + .expect("PBKDF2-SHA-256 cached-state derivation must succeed"); let mut from_oneshot = [0u8; 32]; - Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 100, &mut from_oneshot).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 100, &mut from_oneshot) + .expect("PBKDF2-SHA-256 one-shot derivation must succeed"); assert_eq!(from_state, from_oneshot); // The cached state must be reusable for a second derivation. let mut from_state_again = [0u8; 32]; - state.derive(b"salt2", 50, &mut from_state_again).unwrap(); + state + .derive(b"salt2", 50, &mut from_state_again) + .expect("second PBKDF2-SHA-256 cached-state derivation must succeed"); let mut from_oneshot_again = [0u8; 32]; - Pbkdf2Sha256::derive_key_primitive(b"password", b"salt2", 50, &mut from_oneshot_again).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"password", b"salt2", 50, &mut from_oneshot_again) + .expect("second PBKDF2-SHA-256 one-shot derivation must succeed"); assert_eq!(from_state_again, from_oneshot_again); } @@ -141,10 +165,13 @@ fn pbkdf2_sha256_state_reuse_matches_oneshot() { fn pbkdf2_sha512_state_reuse_matches_oneshot() { let state = Pbkdf2Sha512::new(b"password"); let mut from_state = [0u8; 64]; - state.derive(b"salt", 100, &mut from_state).unwrap(); + state + .derive(b"salt", 100, &mut from_state) + .expect("PBKDF2-SHA-512 cached-state derivation must succeed"); let mut from_oneshot = [0u8; 64]; - Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 100, &mut from_oneshot).unwrap(); + Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 100, &mut from_oneshot) + .expect("PBKDF2-SHA-512 one-shot derivation must succeed"); assert_eq!(from_state, from_oneshot); } diff --git a/tests/pbkdf2_wycheproof.rs b/tests/pbkdf2_wycheproof.rs index 744f73c5..039dae0e 100644 --- a/tests/pbkdf2_wycheproof.rs +++ b/tests/pbkdf2_wycheproof.rs @@ -13,9 +13,10 @@ type Pbkdf2Derive = fn(&[u8], &[u8], u32, &mut [u8]) -> Result<(), Pbkdf2Error>; type Pbkdf2Verify = fn(&[u8], &[u8], u32, &[u8]) -> Result<(), rscrypto::VerificationError>; fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + value + .get(name) + .and_then(Value::as_str) + .expect("Wycheproof string field must exist and contain a string") } fn groups(suite: &Value) -> &[Value] { @@ -44,24 +45,25 @@ fn run_pbkdf2_suite( let tc_id = test["tcId"].as_u64().expect("tcId must be numeric"); let password = decode_hex_vec(field(test, "password")); let salt = decode_hex_vec(field(test, "salt")); - let iterations = test["iterationCount"].as_u64().expect("iterationCount must be numeric") as u32; - let dk_len = test["dkLen"].as_u64().expect("dkLen must be numeric") as usize; + let iterations = u32::try_from(test["iterationCount"].as_u64().expect("iterationCount must be numeric")) + .expect("Wycheproof iterationCount must fit in u32"); + let dk_len = usize::try_from(test["dkLen"].as_u64().expect("dkLen must be numeric")) + .expect("Wycheproof dkLen must fit in usize"); let expected_dk = decode_hex_vec(field(test, "dk")); assert_eq!(expected_dk.len(), dk_len, "{algorithm} tcId {tc_id} dkLen mismatch"); - match field(test, "result") { - "valid" => { - valid += 1; - let mut actual = vec![0u8; dk_len]; - derive(&password, &salt, iterations, &mut actual) - .unwrap_or_else(|err| panic!("{algorithm} tcId {tc_id} failed: {err}")); - assert_eq!(actual, expected_dk, "{algorithm} tcId {tc_id} derived key mismatch"); - - if first_valid_for_negative_check.is_none() { - first_valid_for_negative_check = Some((password, salt, iterations, expected_dk)); - } - } - other => panic!("{algorithm} tcId {tc_id} has unsupported result `{other}`"), + assert_eq!( + field(test, "result"), + "valid", + "{algorithm} tcId {tc_id} has an unsupported result" + ); + valid = valid.strict_add(1); + let mut actual = vec![0u8; dk_len]; + derive(&password, &salt, iterations, &mut actual).expect("known-valid PBKDF2 Wycheproof vector must derive"); + assert_eq!(actual, expected_dk, "{algorithm} tcId {tc_id} derived key mismatch"); + + if first_valid_for_negative_check.is_none() { + first_valid_for_negative_check = Some((password, salt, iterations, expected_dk)); } } } @@ -70,21 +72,12 @@ fn run_pbkdf2_suite( let (password, salt, iterations, expected_dk) = first_valid_for_negative_check.expect("PBKDF2 suite must contain at least one valid vector"); - assert!( - verify(&password, &salt, iterations, &expected_dk).is_ok(), - "{algorithm} rejected a known-good derived key" - ); + verify(&password, &salt, iterations, &expected_dk).expect("PBKDF2 must verify a known-good Wycheproof derived key"); let mut wrong_dk = expected_dk.clone(); wrong_dk[0] ^= 1; - assert!( - verify(&password, &salt, iterations, &wrong_dk).is_err(), - "{algorithm} accepted a corrupted derived key" - ); - assert!( - verify(b"wrong password", &salt, iterations, &expected_dk).is_err(), - "{algorithm} accepted the wrong password" - ); + verify(&password, &salt, iterations, &wrong_dk).expect_err("PBKDF2 must reject a corrupted Wycheproof derived key"); + verify(b"wrong password", &salt, iterations, &expected_dk).expect_err("PBKDF2 must reject the wrong password"); } #[test] diff --git a/tests/phc_external_entropy.rs b/tests/phc_external_entropy.rs index c76f455f..fed326fa 100644 --- a/tests/phc_external_entropy.rs +++ b/tests/phc_external_entropy.rs @@ -15,7 +15,8 @@ const SALT_B64: &str = "AAECAwQFBgcICQoLDA0ODw"; fn argon2_hashes_with_one_caller_entropy_fill() { use rscrypto::{Argon2Params, Argon2idPassword}; - let passwords = Argon2idPassword::new(Argon2Params::new(32, 2, 1).unwrap()).unwrap(); + let params = Argon2Params::new(32, 2, 1).expect("Argon2 entropy-test parameters must be valid"); + let passwords = Argon2idPassword::new(params).expect("Argon2 password service must accept entropy-test parameters"); let fills = Cell::new(0usize); let record = passwords .hash_password_with(PASSWORD, |salt| { @@ -24,7 +25,7 @@ fn argon2_hashes_with_one_caller_entropy_fill() { salt.copy_from_slice(&SALT); Ok::<(), &'static str>(()) }) - .unwrap(); + .expect("Argon2 hashing with caller entropy must succeed"); assert_eq!(fills.get(), 1); assert_eq!(record.split('$').nth(4), Some(SALT_B64)); @@ -39,7 +40,8 @@ fn argon2_hashes_with_one_caller_entropy_fill() { fn scrypt_hashes_with_one_caller_entropy_fill() { use rscrypto::{ScryptParams, ScryptPassword}; - let passwords = ScryptPassword::new(ScryptParams::new(4, 1, 1).unwrap()).unwrap(); + let params = ScryptParams::new(4, 1, 1).expect("scrypt entropy-test parameters must be valid"); + let passwords = ScryptPassword::new(params).expect("scrypt password service must accept entropy-test parameters"); let fills = Cell::new(0usize); let record = passwords .hash_password_with(PASSWORD, |salt| { @@ -48,7 +50,7 @@ fn scrypt_hashes_with_one_caller_entropy_fill() { salt.copy_from_slice(&SALT); Ok::<(), &'static str>(()) }) - .unwrap(); + .expect("scrypt hashing with caller entropy must succeed"); assert_eq!(fills.get(), 1); assert_eq!(record.split('$').nth(3), Some(SALT_B64)); @@ -63,10 +65,11 @@ fn scrypt_hashes_with_one_caller_entropy_fill() { fn argon2_preserves_entropy_errors_without_hashing() { use rscrypto::{Argon2Error, Argon2Params, Argon2idPassword}; - let passwords = Argon2idPassword::new(Argon2Params::new(32, 2, 1).unwrap()).unwrap(); + let params = Argon2Params::new(32, 2, 1).expect("Argon2 entropy-error parameters must be valid"); + let passwords = Argon2idPassword::new(params).expect("Argon2 password service must accept entropy-error parameters"); let error = passwords .hash_password_with(PASSWORD, |_| Err("entropy unavailable")) - .unwrap_err(); + .expect_err("Argon2 must preserve caller entropy failures"); assert_eq!( error, @@ -79,10 +82,11 @@ fn argon2_preserves_entropy_errors_without_hashing() { fn scrypt_preserves_entropy_errors_without_hashing() { use rscrypto::{ScryptError, ScryptParams, ScryptPassword}; - let passwords = ScryptPassword::new(ScryptParams::new(4, 1, 1).unwrap()).unwrap(); + let params = ScryptParams::new(4, 1, 1).expect("scrypt entropy-error parameters must be valid"); + let passwords = ScryptPassword::new(params).expect("scrypt password service must accept entropy-error parameters"); let error = passwords .hash_password_with(PASSWORD, |_| Err("entropy unavailable")) - .unwrap_err(); + .expect_err("scrypt must preserve caller entropy failures"); assert_eq!( error, diff --git a/tests/phc_roundtrip.rs b/tests/phc_roundtrip.rs index 115f1384..af0172e5 100644 --- a/tests/phc_roundtrip.rs +++ b/tests/phc_roundtrip.rs @@ -94,38 +94,59 @@ fn allocations_during(operation: impl FnOnce() -> T) -> (T, usize) { #[test] fn generated_password_records_are_canonical_and_self_verifying() { - let argon2 = Argon2idPassword::new(Argon2Params::new(32, 2, 1).unwrap()).unwrap(); - let argon2_record = argon2.hash_password(b"correct horse battery staple").unwrap(); + let argon2_params = Argon2Params::new(32, 2, 1).expect("Argon2 test parameters must be valid"); + let argon2 = Argon2idPassword::new(argon2_params).expect("Argon2 password service must accept valid parameters"); + let argon2_record = argon2 + .hash_password(b"correct horse battery staple") + .expect("Argon2 password hashing must succeed"); assert!(argon2_record.starts_with("$argon2id$v=19$m=32,t=2,p=1$")); assert_eq!( argon2.verify_password(b"correct horse battery staple", &argon2_record), Ok(PasswordStatus::Current) ); - assert!(argon2.verify_password(b"wrong", &argon2_record).is_err()); - - let scrypt = ScryptPassword::new(ScryptParams::new(4, 1, 1).unwrap()).unwrap(); - let scrypt_record = scrypt.hash_password(b"correct horse battery staple").unwrap(); + argon2 + .verify_password(b"wrong", &argon2_record) + .expect_err("Argon2 verification must reject the wrong password"); + + let scrypt_params = ScryptParams::new(4, 1, 1).expect("scrypt test parameters must be valid"); + let scrypt = ScryptPassword::new(scrypt_params).expect("scrypt password service must accept valid parameters"); + let scrypt_record = scrypt + .hash_password(b"correct horse battery staple") + .expect("scrypt password hashing must succeed"); assert!(scrypt_record.starts_with("$scrypt$ln=4,r=1,p=1$")); assert_eq!( scrypt.verify_password(b"correct horse battery staple", &scrypt_record), Ok(PasswordStatus::Current) ); - assert!(scrypt.verify_password(b"wrong", &scrypt_record).is_err()); + scrypt + .verify_password(b"wrong", &scrypt_record) + .expect_err("scrypt verification must reject the wrong password"); } #[test] fn verifier_reports_accepted_stale_profiles() { - let old_argon2 = Argon2idPassword::new(Argon2Params::new(32, 2, 1).unwrap()).unwrap(); - let argon2_record = old_argon2.hash_password(b"password").unwrap(); - let current_argon2 = Argon2idPassword::new(Argon2Params::new(40, 2, 1).unwrap()).unwrap(); + let old_argon2_params = Argon2Params::new(32, 2, 1).expect("old Argon2 profile must be valid"); + let old_argon2 = + Argon2idPassword::new(old_argon2_params).expect("old Argon2 password service must accept its profile"); + let argon2_record = old_argon2 + .hash_password(b"password") + .expect("old Argon2 profile must produce a password record"); + let current_argon2_params = Argon2Params::new(40, 2, 1).expect("current Argon2 profile must be valid"); + let current_argon2 = + Argon2idPassword::new(current_argon2_params).expect("current Argon2 password service must accept its profile"); assert_eq!( current_argon2.verify_password(b"password", &argon2_record), Ok(PasswordStatus::NeedsRehash) ); - let old_scrypt = ScryptPassword::new(ScryptParams::new(4, 1, 1).unwrap()).unwrap(); - let scrypt_record = old_scrypt.hash_password(b"password").unwrap(); - let current_scrypt = ScryptPassword::new(ScryptParams::new(5, 1, 1).unwrap()).unwrap(); + let old_scrypt_params = ScryptParams::new(4, 1, 1).expect("old scrypt profile must be valid"); + let old_scrypt = ScryptPassword::new(old_scrypt_params).expect("old scrypt password service must accept its profile"); + let scrypt_record = old_scrypt + .hash_password(b"password") + .expect("old scrypt profile must produce a password record"); + let current_scrypt_params = ScryptParams::new(5, 1, 1).expect("current scrypt profile must be valid"); + let current_scrypt = + ScryptPassword::new(current_scrypt_params).expect("current scrypt password service must accept its profile"); assert_eq!( current_scrypt.verify_password(b"password", &scrypt_record), Ok(PasswordStatus::NeedsRehash) @@ -134,7 +155,9 @@ fn verifier_reports_accepted_stale_profiles() { #[test] fn every_rejected_phc_class_allocates_nothing() { - let argon2 = Argon2idPassword::new(Argon2Params::new(32, 2, 1).unwrap()).unwrap(); + let argon2_params = Argon2Params::new(32, 2, 1).expect("Argon2 rejection-test parameters must be valid"); + let argon2 = + Argon2idPassword::new(argon2_params).expect("Argon2 password service must accept rejection-test parameters"); let oversized = "x".repeat(1_025); let argon2_rejections = [ oversized.as_str(), @@ -148,11 +171,13 @@ fn every_rejected_phc_class_allocates_nothing() { ]; for encoded in argon2_rejections { let (result, allocations) = allocations_during(|| argon2.verify_password(b"password", encoded)); - assert!(result.is_err(), "rejected Argon2 PHC: {encoded}"); + result.expect_err("known-invalid Argon2 PHC record must be rejected"); assert_eq!(allocations, 0, "rejected Argon2 PHC allocated: {encoded}"); } - let scrypt = ScryptPassword::new(ScryptParams::new(4, 1, 1).unwrap()).unwrap(); + let scrypt_params = ScryptParams::new(4, 1, 1).expect("scrypt rejection-test parameters must be valid"); + let scrypt = + ScryptPassword::new(scrypt_params).expect("scrypt password service must accept rejection-test parameters"); let scrypt_rejections = [ oversized.as_str(), "not-a-phc-record", @@ -165,34 +190,42 @@ fn every_rejected_phc_class_allocates_nothing() { ]; for encoded in scrypt_rejections { let (result, allocations) = allocations_during(|| scrypt.verify_password(b"password", encoded)); - assert!(result.is_err(), "rejected scrypt PHC: {encoded}"); + result.expect_err("known-invalid scrypt PHC record must be rejected"); assert_eq!(allocations, 0, "rejected scrypt PHC allocated: {encoded}"); } } #[test] fn public_verifiers_reject_noncanonical_and_cross_algorithm_records() { - let argon2 = Argon2idPassword::new(Argon2Params::new(32, 2, 1).unwrap()).unwrap(); - let scrypt = ScryptPassword::new(ScryptParams::new(4, 1, 1).unwrap()).unwrap(); - let valid_argon2 = argon2.hash_password(b"password").unwrap(); - let valid_scrypt = scrypt.hash_password(b"password").unwrap(); - - assert!(argon2.verify_password(b"password", &valid_scrypt).is_err()); - assert!(scrypt.verify_password(b"password", &valid_argon2).is_err()); - assert!( - argon2 - .verify_password( - b"password", - "$argon2id$v=19$t=2,m=32,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", - ) - .is_err() - ); - assert!( - scrypt - .verify_password( - b"password", - "$scrypt$r=1,ln=4,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", - ) - .is_err() - ); + let argon2_params = Argon2Params::new(32, 2, 1).expect("Argon2 cross-algorithm parameters must be valid"); + let argon2 = + Argon2idPassword::new(argon2_params).expect("Argon2 password service must accept cross-algorithm parameters"); + let scrypt_params = ScryptParams::new(4, 1, 1).expect("scrypt cross-algorithm parameters must be valid"); + let scrypt = + ScryptPassword::new(scrypt_params).expect("scrypt password service must accept cross-algorithm parameters"); + let valid_argon2 = argon2 + .hash_password(b"password") + .expect("Argon2 must produce a cross-algorithm fixture"); + let valid_scrypt = scrypt + .hash_password(b"password") + .expect("scrypt must produce a cross-algorithm fixture"); + + argon2 + .verify_password(b"password", &valid_scrypt) + .expect_err("Argon2 must reject a scrypt record"); + scrypt + .verify_password(b"password", &valid_argon2) + .expect_err("scrypt must reject an Argon2 record"); + argon2 + .verify_password( + b"password", + "$argon2id$v=19$t=2,m=32,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ) + .expect_err("Argon2 must reject noncanonical parameter order"); + scrypt + .verify_password( + b"password", + "$scrypt$r=1,ln=4,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ) + .expect_err("scrypt must reject noncanonical parameter order"); } diff --git a/tests/platform_amx_permission.rs b/tests/platform_amx_permission.rs index d8837a15..d72d4e92 100644 --- a/tests/platform_amx_permission.rs +++ b/tests/platform_amx_permission.rs @@ -5,7 +5,6 @@ target_os = "linux", not(miri) ))] -#![allow(unsafe_code)] use rscrypto::platform::{self, caps::x86}; @@ -18,7 +17,6 @@ const ARCH_REQ_XCOMP_PERM: usize = 0x1023; const XFEATURE_XTILEDATA: usize = 18; const XCOMP_TILE_MASK: u64 = (1 << 17) | (1 << 18); -#[allow(unused_unsafe)] fn cpu_supports_amx_tile() -> bool { // MSRV: CPUID is unsafe on Rust 1.91 but safe on the pinned nightly. // SAFETY: CPUID is a non-privileged x86-64 identification instruction. diff --git a/tests/platform_override_race.rs b/tests/platform_override_race.rs index b6f78f72..257b85ce 100644 --- a/tests/platform_override_race.rs +++ b/tests/platform_override_race.rs @@ -10,14 +10,15 @@ fn concurrent_override_writers_are_serialized() { for _ in 0..8 { scope.spawn(|| { for _ in 0..128 { - expert::try_set_override(Some(Detected::portable())).unwrap(); - expert::try_set_override(None).unwrap(); + expert::try_set_override(Some(Detected::portable())) + .expect("portable override must be accepted before detection"); + expert::try_set_override(None).expect("override must be clearable before detection"); } }); } }); - expert::try_set_override(None).unwrap(); + expert::try_set_override(None).expect("override must be clearable after concurrent updates"); } #[test] @@ -39,13 +40,15 @@ fn concurrent_detection_and_override_child() { }); barrier.wait(); - let setter_result = setter.join().unwrap(); - let detected = detector.join().unwrap(); + let setter_result = setter.join().expect("override thread must not panic"); + let detected = detector.join().expect("detection thread must not panic"); - match setter_result { - Ok(()) => assert_eq!(detected, Detected::portable()), - Err(expert::OverrideError::AlreadyInitialized) => {} - Err(error) => panic!("unexpected override result: {error:?}"), + assert!( + matches!(setter_result, Ok(()) | Err(expert::OverrideError::AlreadyInitialized)), + "concurrent override returned an invalid result: {setter_result:?}" + ); + if setter_result == Ok(()) { + assert_eq!(detected, Detected::portable()); } }); } @@ -53,7 +56,7 @@ fn concurrent_detection_and_override_child() { #[test] #[cfg(not(miri))] fn concurrent_detection_and_override_are_linearizable() { - let executable = std::env::current_exe().unwrap(); + let executable = std::env::current_exe().expect("test executable path must be available"); for _ in 0..32 { let status = std::process::Command::new(&executable) @@ -62,7 +65,7 @@ fn concurrent_detection_and_override_are_linearizable() { .arg("--quiet") .env(CHILD_MODE, "1") .status() - .unwrap(); + .expect("race child process must start"); assert!(status.success(), "race child failed with {status}"); } } diff --git a/tests/platform_override_validation.rs b/tests/platform_override_validation.rs index c4f82ab0..e190983f 100644 --- a/tests/platform_override_validation.rs +++ b/tests/platform_override_validation.rs @@ -18,7 +18,7 @@ fn safe_override_rejects_impossible_caps_and_still_allows_portable() { ); let portable = Detected::portable(); - expert::try_set_override(Some(portable)).unwrap(); + expert::try_set_override(Some(portable)).expect("portable override must be accepted before detection"); assert!(expert::has_override()); assert_eq!(platform::get(), portable); @@ -33,7 +33,7 @@ fn safe_override_rejects_impossible_caps_and_still_allows_portable() { #[cfg(miri)] { - expert::try_set_override(None).unwrap(); + expert::try_set_override(None).expect("Miri override must remain mutable"); assert!(!expert::has_override()); } } diff --git a/tests/poly1305_vectors.rs b/tests/poly1305_vectors.rs index 45b3a886..35009dcc 100644 --- a/tests/poly1305_vectors.rs +++ b/tests/poly1305_vectors.rs @@ -3,7 +3,9 @@ use rscrypto::{Poly1305, Poly1305OneTimeKey, Poly1305Tag}; mod common; -use common::decode_hex_array; +#[path = "common/array.rs"] +mod hex_array; +use hex_array::decode_hex_array; #[test] fn poly1305_matches_rfc_8439_section_2_5_2() { @@ -39,7 +41,7 @@ fn poly1305_verify_rejects_corrupted_tag() { corrupted[7] ^= 0x80; let corrupted = Poly1305Tag::from_bytes(corrupted); - assert!(Poly1305::verify_once(key, message, &corrupted).is_err()); + Poly1305::verify_once(key, message, &corrupted).expect_err("corrupted Poly1305 tag must be rejected"); } #[test] @@ -48,8 +50,8 @@ fn poly1305_try_generate_with_uses_fallible_fill() { out.fill(0x11); Ok::<(), ()>(()) }) - .unwrap(); + .expect("deterministic entropy callback must generate a Poly1305 key"); assert_eq!(key.as_bytes(), &[0x11; Poly1305OneTimeKey::LENGTH]); - assert!(Poly1305OneTimeKey::try_generate_with(|_| Err("rng")).is_err()); + Poly1305OneTimeKey::try_generate_with(|_| Err("rng")).expect_err("entropy-source failure must be returned"); } diff --git a/tests/portable_fallback.rs b/tests/portable_fallback.rs index 1ac59305..b10372f5 100644 --- a/tests/portable_fallback.rs +++ b/tests/portable_fallback.rs @@ -187,7 +187,7 @@ fn crc64_empty_input() { fn crc32_various_sizes() { // Just verify that different sizes don't panic and produce consistent results for size in [1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 255, 256, 1024] { - let data: Vec = (0..size).map(|i| (i & 0xFF) as u8).collect(); + let data: Vec = (0..size).map(|i: usize| i.to_le_bytes()[0]).collect(); let oneshot = Crc32::checksum(&data); @@ -205,7 +205,7 @@ fn crc32_various_sizes() { #[test] fn crc64_various_sizes() { for size in [1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 255, 256, 1024] { - let data: Vec = (0..size).map(|i| (i & 0xFF) as u8).collect(); + let data: Vec = (0..size).map(|i: usize| i.to_le_bytes()[0]).collect(); let oneshot = Crc64::checksum(&data); diff --git a/tests/rapidhash_differential.rs b/tests/rapidhash_differential.rs index 3cf0addb..e0cac670 100644 --- a/tests/rapidhash_differential.rs +++ b/tests/rapidhash_differential.rs @@ -23,7 +23,7 @@ fn collection_state_schedules_match_rapidhash() { for seed in [0, 1, u64::MAX, 0x243f_6a88_85a3_08d3] { for len in lengths { let data: Vec = (0..len) - .map(|index| index.wrapping_mul(131).wrapping_add(17) as u8) + .map(|index| index.wrapping_mul(131).wrapping_add(17).to_le_bytes()[0]) .collect(); let deterministic = RapidSeededState::new(seed); @@ -37,7 +37,7 @@ fn collection_state_schedules_match_rapidhash() { out.copy_from_slice(&seed.to_le_bytes()); Ok::<_, ()>(()) }) - .unwrap(); + .expect("deterministic entropy callback must initialize RapidRandomState"); let secrets = rapidhash::v3::RapidSecrets::seed(seed); assert_eq!( hash_bytes(&randomized, &data), diff --git a/tests/root_surface.rs b/tests/root_surface.rs index 6480602c..266339c3 100644 --- a/tests/root_surface.rs +++ b/tests/root_surface.rs @@ -1,9 +1,11 @@ -#![allow(unused_imports)] - #[cfg(feature = "aead")] use rscrypto::Aead; -#[cfg(any(feature = "hmac", feature = "hmac-sha3", feature = "kmac"))] +#[cfg(feature = "hmac")] use rscrypto::Mac; +#[cfg(all(feature = "rsa", feature = "getrandom"))] +use rscrypto::RsaJwtAlgorithm; +#[cfg(all(feature = "hashes", any(feature = "std", feature = "diag")))] +use rscrypto::Sha256; #[cfg(feature = "aead")] use rscrypto::aead::expert::AeadWithNonce; #[cfg(all(feature = "aead", feature = "diag"))] @@ -21,7 +23,7 @@ use rscrypto::aead::{ use rscrypto::auth::HkdfOutputLengthError; #[cfg(all(feature = "checksums", feature = "alloc"))] use rscrypto::checksum::buffered::BufferedCrc32C; -#[cfg(feature = "checksums")] +#[cfg(all(feature = "checksums", feature = "diag"))] use rscrypto::checksum::config::{ Crc16Config, Crc16Force, Crc24Config, Crc24Force, Crc32Config, Crc32Force, Crc64Config, Crc64Force, }; @@ -39,11 +41,12 @@ use rscrypto::hashes::introspect::{ use rscrypto::hashes::{DigestReader, DigestWriter}; #[cfg(feature = "hashes")] use rscrypto::{ - AsconCxof128, AsconCxof128Reader, AsconHash256, AsconXof, AsconXofReader, Blake3, Blake3XofReader, Cshake128, - Cshake128XofReader, Cshake256, Cshake256XofReader, Digest, FastHash, RapidHash64, RapidHasher, RapidRandomState, - RapidSeededState, RapidStreamHasher, Sha3_224, Sha3_256, Sha3_384, Sha3_512, Sha224, Sha256, Sha384, Sha512, - Sha512_256, Shake128, Shake128XofReader, Shake256, Shake256XofReader, Xof, Xxh3, Xxh3_128, + AsconCxof128, AsconCxof128Reader, AsconXof, AsconXofReader, Blake3, Blake3XofReader, Cshake128, Cshake128XofReader, + Cshake256, Cshake256XofReader, Digest, FastHash, RapidHash64, RapidHasher, RapidRandomState, RapidSeededState, + RapidStreamHasher, Shake128, Shake128XofReader, Shake256, Shake256XofReader, Xof, Xxh3, }; +#[cfg(all(feature = "hashes", feature = "diag"))] +use rscrypto::{AsconHash256, Sha3_224, Sha3_256, Sha3_384, Sha3_512, Sha224, Sha384, Sha512, Sha512_256, Xxh3_128}; #[cfg(feature = "checksums")] use rscrypto::{Checksum, ChecksumCombine, Crc16Ccitt, Crc16Ibm, Crc24OpenPgp, Crc32, Crc32C, Crc64, Crc64Nvme}; #[cfg(feature = "ed25519")] @@ -71,7 +74,7 @@ use rscrypto::{ use rscrypto::{Poly1305, Poly1305OneTimeKey, Poly1305Tag}; #[cfg(feature = "rsa")] use rscrypto::{ - RsaEncryptionError, RsaJwtAlgorithm, RsaJwtVerifier, RsaKeyError, RsaKeyGenerationError, RsaOaepProfile, + RsaBlindingPair, RsaEncryptionError, RsaJwtVerifier, RsaKeyError, RsaKeyGenerationError, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPrivateOpError, RsaPrivateScratch, RsaProtocolAlgorithmError, RsaPssProfile, RsaPublicExponent, RsaPublicExponentPolicy, RsaPublicKey, RsaPublicKeyPolicy, RsaPublicOpError, RsaPublicScratch, RsaSignatureProfile, RsaTlsSignatureSchemes, @@ -81,7 +84,7 @@ use rscrypto::{VerificationError, ct}; #[cfg(feature = "x25519")] use rscrypto::{X25519Error, X25519PublicKey, X25519SecretKey, X25519SharedSecret}; -#[cfg(feature = "rsa")] +#[cfg(all(feature = "rsa", feature = "getrandom"))] fn fill_rsa_random_with(byte: u8) -> impl FnMut(&mut [u8]) -> Result<(), RsaEncryptionError> { move |out| { out.fill(byte); @@ -89,7 +92,7 @@ fn fill_rsa_random_with(byte: u8) -> impl FnMut(&mut [u8]) -> Result<(), RsaEncr } } -#[cfg(feature = "rsa")] +#[cfg(all(feature = "rsa", feature = "getrandom"))] fn fill_rsa_random_from(bytes: &[u8]) -> impl FnMut(&mut [u8]) -> Result<(), RsaEncryptionError> + '_ { let mut offset = 0usize; move |out| { @@ -105,7 +108,7 @@ fn fill_rsa_random_from(bytes: &[u8]) -> impl FnMut(&mut [u8]) -> Result<(), Rsa #[test] fn root_surface_core_exports_compile() { - let _ = VerificationError::new(); + let _error = VerificationError::new(); let mut secret = [0x5a; 8]; ct::zeroize(&mut secret); assert_eq!(secret, [0; 8]); @@ -125,22 +128,22 @@ fn root_surface_aead_exports_compile() { let nonce256 = Nonce256::from_bytes([0x44; Nonce256::LENGTH]); assert_eq!(nonce256.as_bytes().len(), Nonce256::LENGTH); - let _ = AeadBufferError::new(); - let _ = SealError::buffer(); - let _ = SealError::too_large(); - let _ = OpenError::buffer(); - let _ = OpenError::too_large(); - let _ = OpenError::verification(); + let _buffer_error = AeadBufferError::new(); + let _seal_buffer_error = SealError::buffer(); + let _seal_length_error = SealError::too_large(); + let _open_buffer_error = OpenError::buffer(); + let _open_length_error = OpenError::too_large(); + let _verification_error = OpenError::verification(); #[cfg(feature = "diag")] { - let _ = AeadDispatchInfo::current(); - let _ = aes256gcm_backend(); - let _ = aes256gcmsiv_backend(); - let _ = chacha20poly1305_backend(); - let _ = xchacha20poly1305_backend(); - let _ = aegis256_backend(); - let _ = ascon_aead128_backend(); + let _dispatch = AeadDispatchInfo::current(); + let _aes256gcm_backend = aes256gcm_backend(); + let _aes256gcmsiv_backend = aes256gcmsiv_backend(); + let _chacha20poly1305_backend = chacha20poly1305_backend(); + let _xchacha20poly1305_backend = xchacha20poly1305_backend(); + let _aegis256_backend = aegis256_backend(); + let _ascon_aead128_backend = ascon_aead128_backend(); } fn assert_aead_trait() {} @@ -149,20 +152,26 @@ fn root_surface_aead_exports_compile() { let key = XChaCha20Poly1305Key::from_bytes([0x44; XChaCha20Poly1305::KEY_SIZE]); let cipher = XChaCha20Poly1305::new(&key); let mut sealed = [0u8; 20]; - cipher.encrypt(&nonce192, b"aad", b"test", &mut sealed).unwrap(); - let _ = XChaCha20Poly1305Tag::from_bytes([0u8; XChaCha20Poly1305Tag::LENGTH]); + cipher + .encrypt(&nonce192, b"aad", b"test", &mut sealed) + .expect("valid XChaCha20-Poly1305 input must encrypt"); + let _tag = XChaCha20Poly1305Tag::from_bytes([0u8; XChaCha20Poly1305Tag::LENGTH]); let key = ChaCha20Poly1305Key::from_bytes([0x55; ChaCha20Poly1305::KEY_SIZE]); let cipher = ChaCha20Poly1305::new(&key); let mut sealed = [0u8; 20]; - cipher.encrypt(&nonce96, b"aad", b"test", &mut sealed).unwrap(); - let _ = ChaCha20Poly1305Tag::from_bytes([0u8; ChaCha20Poly1305Tag::LENGTH]); + cipher + .encrypt(&nonce96, b"aad", b"test", &mut sealed) + .expect("valid ChaCha20-Poly1305 input must encrypt"); + let _tag = ChaCha20Poly1305Tag::from_bytes([0u8; ChaCha20Poly1305Tag::LENGTH]); let key = Aegis256Key::from_bytes([0x66; Aegis256::KEY_SIZE]); let cipher = Aegis256::new(&key); let mut sealed = [0u8; 20]; - cipher.encrypt(&nonce256, b"aad", b"test", &mut sealed).unwrap(); - let _ = Aegis256Tag::from_bytes([0u8; Aegis256Tag::LENGTH]); + cipher + .encrypt(&nonce256, b"aad", b"test", &mut sealed) + .expect("valid AEGIS-256 input must encrypt"); + let _tag = Aegis256Tag::from_bytes([0u8; Aegis256Tag::LENGTH]); } #[test] @@ -174,26 +183,29 @@ fn root_surface_mac_exports_compile() { let tag = HmacSha256::mac(key, data); let tag384 = HmacSha384::mac(key, data); let tag512 = HmacSha512::mac(key, data); - let _ = HmacSha256Tag::from_bytes(tag.to_bytes()); - let _ = HmacSha384Tag::from_bytes(tag384.to_bytes()); - let _ = HmacSha512Tag::from_bytes(tag512.to_bytes()); + let _tag256 = HmacSha256Tag::from_bytes(tag.to_bytes()); + let _tag384 = HmacSha384Tag::from_bytes(tag384.to_bytes()); + let _tag512 = HmacSha512Tag::from_bytes(tag512.to_bytes()); let mut mac = HmacSha256::new(key); mac.update(data); assert!(tag.ct_eq(&mac.finalize()).declassify()); - assert!(mac.verify(&tag).is_ok()); - let prefix = *tag.as_bytes().first_chunk::<8>().unwrap(); - assert!(HmacSha256::verify_truncated_tag_64(key, data, &prefix).is_ok()); + mac.verify(&tag).expect("matching HMAC-SHA-256 tag must verify"); + let prefix = *tag + .as_bytes() + .first_chunk::<8>() + .expect("an HMAC-SHA-256 tag must contain an eight-byte prefix"); + HmacSha256::verify_truncated_tag_64(key, data, &prefix).expect("matching truncated HMAC-SHA-256 tag must verify"); let mut mac384 = HmacSha384::new(key); mac384.update(data); assert!(tag384.ct_eq(&mac384.finalize()).declassify()); - assert!(mac384.verify(&tag384).is_ok()); + mac384.verify(&tag384).expect("matching HMAC-SHA-384 tag must verify"); let mut mac512 = HmacSha512::new(key); mac512.update(data); assert!(tag512.ct_eq(&mac512.finalize()).declassify()); - assert!(mac512.verify(&tag512).is_ok()); + mac512.verify(&tag512).expect("matching HMAC-SHA-512 tag must verify"); } #[test] @@ -206,15 +218,15 @@ fn root_surface_hmac_sha3_exports_compile() { let tag256 = HmacSha3_256::mac(key, data); let tag384 = HmacSha3_384::mac(key, data); let tag512 = HmacSha3_512::mac(key, data); - let _ = HmacSha3_224Tag::from_bytes(tag224.to_bytes()); - let _ = HmacSha3_256Tag::from_bytes(tag256.to_bytes()); - let _ = HmacSha3_384Tag::from_bytes(tag384.to_bytes()); - let _ = HmacSha3_512Tag::from_bytes(tag512.to_bytes()); - - assert!(HmacSha3_224::verify_tag(key, data, &tag224).is_ok()); - assert!(HmacSha3_256::verify_tag(key, data, &tag256).is_ok()); - assert!(HmacSha3_384::verify_tag(key, data, &tag384).is_ok()); - assert!(HmacSha3_512::verify_tag(key, data, &tag512).is_ok()); + let _tag224 = HmacSha3_224Tag::from_bytes(tag224.to_bytes()); + let _tag256 = HmacSha3_256Tag::from_bytes(tag256.to_bytes()); + let _tag384 = HmacSha3_384Tag::from_bytes(tag384.to_bytes()); + let _tag512 = HmacSha3_512Tag::from_bytes(tag512.to_bytes()); + + HmacSha3_224::verify_tag(key, data, &tag224).expect("matching HMAC-SHA3-224 tag must verify"); + HmacSha3_256::verify_tag(key, data, &tag256).expect("matching HMAC-SHA3-256 tag must verify"); + HmacSha3_384::verify_tag(key, data, &tag384).expect("matching HMAC-SHA3-384 tag must verify"); + HmacSha3_512::verify_tag(key, data, &tag512).expect("matching HMAC-SHA3-512 tag must verify"); } #[test] @@ -224,19 +236,34 @@ fn root_surface_kdf_exports_compile() { let mut out = [0u8; 32]; let hkdf = HkdfSha256::new(b"salt", key); - hkdf.expand(b"info", &mut out).unwrap(); - assert_eq!(out, HkdfSha256::derive_array::<32>(b"salt", key, b"info").unwrap()); + hkdf + .expand(b"info", &mut out) + .expect("32-byte HKDF-SHA-256 output must fit"); + assert_eq!( + out, + HkdfSha256::derive_array::<32>(b"salt", key, b"info").expect("32-byte HKDF-SHA-256 output must fit") + ); let mut out384 = [0u8; 48]; let hkdf384 = HkdfSha384::new(b"salt", key); - hkdf384.expand(b"info", &mut out384).unwrap(); - assert_eq!(out384, HkdfSha384::derive_array::<48>(b"salt", key, b"info").unwrap()); + hkdf384 + .expand(b"info", &mut out384) + .expect("48-byte HKDF-SHA-384 output must fit"); + assert_eq!( + out384, + HkdfSha384::derive_array::<48>(b"salt", key, b"info").expect("48-byte HKDF-SHA-384 output must fit") + ); let mut out512 = [0u8; 64]; let hkdf512 = HkdfSha512::new(b"salt", key); - hkdf512.expand(b"info", &mut out512).unwrap(); - assert_eq!(out512, HkdfSha512::derive_array::<64>(b"salt", key, b"info").unwrap()); - let _ = HkdfOutputLengthError::new(); + hkdf512 + .expand(b"info", &mut out512) + .expect("64-byte HKDF-SHA-512 output must fit"); + assert_eq!( + out512, + HkdfSha512::derive_array::<64>(b"salt", key, b"info").expect("64-byte HKDF-SHA-512 output must fit") + ); + let _length_error = HkdfOutputLengthError::new(); } #[test] @@ -248,13 +275,13 @@ fn root_surface_kmac_exports_compile() { let mut kmac128 = Kmac128::new(key, b"svc=v1"); kmac128.update(data); kmac128.finalize_into(&mut out128); - assert!(Kmac128::verify_tag(key, b"svc=v1", data, &out128).is_ok()); + Kmac128::verify_tag(key, b"svc=v1", data, &out128).expect("matching KMAC128 tag must verify"); let mut out = [0u8; 32]; let mut kmac = Kmac256::new(key, b"svc=v1"); kmac.update(data); kmac.finalize_into(&mut out); - assert!(Kmac256::verify_tag(key, b"svc=v1", data, &out).is_ok()); + Kmac256::verify_tag(key, b"svc=v1", data, &out).expect("matching KMAC256 tag must verify"); } #[test] @@ -262,10 +289,10 @@ fn root_surface_kmac_exports_compile() { fn root_surface_poly1305_exports_compile() { let key = Poly1305OneTimeKey::from_bytes([0x33; Poly1305OneTimeKey::LENGTH]); let tag = Poly1305::authenticate_once(key, b"root-surface-poly1305"); - let _ = Poly1305Tag::from_bytes(tag.to_bytes()); + let _tag = Poly1305Tag::from_bytes(tag.to_bytes()); let key = Poly1305OneTimeKey::from_bytes([0x33; Poly1305OneTimeKey::LENGTH]); - assert!(Poly1305::verify_once(key, b"root-surface-poly1305", &tag).is_ok()); + Poly1305::verify_once(key, b"root-surface-poly1305", &tag).expect("matching Poly1305 tag must verify"); } #[test] @@ -328,13 +355,15 @@ fn root_surface_mlkem_exports_compile() { fn root_surface_signature_exports_compile() { let secret = Ed25519SecretKey::from_bytes([7u8; Ed25519SecretKey::LENGTH]); let keypair = Ed25519Keypair::from_secret_key(secret.duplicate_secret()); - let public = keypair.public_key(); - let signature = keypair.sign(b"root-surface-ed25519"); + let public: Ed25519PublicKey = keypair.public_key(); + let signature: Ed25519Signature = keypair.sign(b"root-surface-ed25519"); assert_eq!(secret.as_bytes().len(), 32); assert_eq!(public.as_bytes().len(), 32); assert_eq!(signature.as_bytes().len(), 64); - assert!(public.verify(b"root-surface-ed25519", &signature).is_ok()); + public + .verify(b"root-surface-ed25519", &signature) + .expect("matching Ed25519 signature must verify"); } #[test] @@ -343,24 +372,25 @@ fn root_surface_rsa_exports_compile() { let policy = RsaPublicKeyPolicy::legacy_verification().allow_legacy_small_exponents(); assert_eq!(policy.min_modulus_bits(), 2048); assert_eq!(policy.max_modulus_bits(), 8192); - let _ = RsaPublicExponentPolicy::Common65537; - let _ = RsaKeyError::InvalidModulus; - let _ = RsaKeyGenerationError::InvalidModulusBits; - let _ = RsaEncryptionError::InvalidLength; - let _ = RsaPrivateOpError::InvalidLength; - let _ = RsaPublicOpError::RepresentativeOutOfRange; - let _ = RsaProtocolAlgorithmError::UnsupportedAlgorithm; - let _ = RsaOaepProfile::Sha256; - let _ = RsaPssProfile::Sha256; - let _ = RsaPkcs1v15Profile::Sha256; - let _: Option = None; - let _: Option = None; - let _: Option = None; - let _: Option> = None; - let _: Option = None; - let _: Option> = None; - let _: Option = None; - let _: Option = None; + let _exponent_policy = RsaPublicExponentPolicy::Common65537; + let _key_error = RsaKeyError::InvalidModulus; + let _generation_error = RsaKeyGenerationError::InvalidModulusBits; + let _encryption_error = RsaEncryptionError::InvalidLength; + let _private_op_error = RsaPrivateOpError::InvalidLength; + let _public_op_error = RsaPublicOpError::RepresentativeOutOfRange; + let _protocol_error = RsaProtocolAlgorithmError::UnsupportedAlgorithm; + let _oaep_profile = RsaOaepProfile::Sha256; + let _pss_profile = RsaPssProfile::Sha256; + let _pkcs1v15_profile = RsaPkcs1v15Profile::Sha256; + let _public_exponent: Option = None; + let _public_key: Option = None; + let _public_scratch: Option = None; + let _jwt_verifier: Option> = None; + let _private_key: Option = None; + let _private_key_parts: Option> = None; + let _blinding_pair = RsaBlindingPair::new(&[], &[]); + let _private_scratch: Option = None; + let _x509_public_key: Option = None; assert_eq!( RsaSignatureProfile::pss(RsaPssProfile::Sha256).pss_parts(), Some((RsaPssProfile::Sha256, 32)) @@ -370,23 +400,20 @@ fn root_surface_rsa_exports_compile() { Some(RsaPkcs1v15Profile::Sha384) ); assert_eq!( - RsaSignatureProfile::from_tls13_signature_scheme(0x0804).unwrap(), + RsaSignatureProfile::from_tls13_signature_scheme(0x0804) + .expect("TLS 1.3 rsa_pss_rsae_sha256 must map to an RSA signature profile"), RsaSignatureProfile::pss(RsaPssProfile::Sha256) ); - assert!( - RsaX509PublicKeyAlgorithm::RsaPss - .permits_signature_profile(RsaSignatureProfile::pss(RsaPssProfile::Sha256)) - .is_ok() - ); - assert!( - RsaX509PublicKeyAlgorithm::RsaEncryption - .signature_profile_from_tls13_signature_scheme(0x0804) - .is_ok() - ); + RsaX509PublicKeyAlgorithm::RsaPss + .permits_signature_profile(RsaSignatureProfile::pss(RsaPssProfile::Sha256)) + .expect("an RSA-PSS key must permit the SHA-256 PSS profile"); + RsaX509PublicKeyAlgorithm::RsaEncryption + .signature_profile_from_tls13_signature_scheme(0x0804) + .expect("an rsaEncryption key must accept TLS 1.3 rsa_pss_rsae_sha256"); let advertised = RsaX509PublicKeyAlgorithm::RsaEncryption.advertised_tls13_signature_schemes(); assert_eq!(advertised.len(), 3); assert!(advertised.contains(0x0804)); - let _ = RsaTlsSignatureSchemes::MAX_LEN; + let _maximum_scheme_count = RsaTlsSignatureSchemes::MAX_LEN; } #[test] @@ -403,9 +430,11 @@ fn root_surface_rsa_generated_key_end_to_end() { ]; let policy = RsaPublicKeyPolicy::legacy_verification(); - let key = RsaPrivateKey::generate_with_policy(2048, &policy).unwrap(); + let key = + RsaPrivateKey::generate_with_policy(2048, &policy).expect("the supported 2048-bit policy must generate an RSA key"); let public_key = key.public_key(); - let x509_key = RsaX509PublicKey::from_spki_der_with_policy(&public_key.to_spki_der(), &policy).unwrap(); + let x509_key = RsaX509PublicKey::from_spki_der_with_policy(&public_key.to_spki_der(), &policy) + .expect("the generated public key must round-trip through X.509 SPKI"); let message = b"root-surface-rsa-generated-key"; let mut private_scratch = key.private_scratch(); let mut public_scratch = public_key.public_scratch(); @@ -416,29 +445,31 @@ fn root_surface_rsa_generated_key_end_to_end() { assert_eq!(format!("{pkcs1_der:?}"), "SecretVec(****)"); assert_eq!( RsaPrivateKey::from_pkcs1_der_with_policy(&pkcs1_der, &policy) - .unwrap() + .expect("the generated private key must round-trip through PKCS#1 DER") .public_key(), public_key ); assert_eq!( RsaPrivateKey::from_pkcs8_der_with_policy(&pkcs8_der, &policy) - .unwrap() + .expect("the generated private key must round-trip through PKCS#8 DER") .public_key(), public_key ); assert_eq!( - RsaPublicKey::from_pkcs1_der_with_policy(&public_key.to_pkcs1_der(), &policy).unwrap(), + RsaPublicKey::from_pkcs1_der_with_policy(&public_key.to_pkcs1_der(), &policy) + .expect("the generated public key must round-trip through PKCS#1 DER"), *public_key ); assert_eq!( - RsaPublicKey::from_spki_der_with_policy(&public_key.to_spki_der(), &policy).unwrap(), + RsaPublicKey::from_spki_der_with_policy(&public_key.to_spki_der(), &policy) + .expect("the generated public key must round-trip through SPKI DER"), *public_key ); let mut unprotected_pkcs1 = key.to_pkcs1_der().into_unprotected_vec(); assert_eq!( RsaPrivateKey::from_pkcs1_der_with_policy(&unprotected_pkcs1, &policy) - .unwrap() + .expect("the unprotected PKCS#1 bytes must parse before zeroization") .public_key(), public_key ); @@ -451,54 +482,62 @@ fn root_surface_rsa_generated_key_end_to_end() { (RsaPkcs1v15Profile::Sha512, RsaPssProfile::Sha512), ] { let pkcs1v15_profile = RsaSignatureProfile::pkcs1v15(pkcs1v15_profile); - key.sign_signature(pkcs1v15_profile, message, &mut signature).unwrap(); + key + .sign_signature(pkcs1v15_profile, message, &mut signature) + .expect("the generated key must produce a PKCS#1 v1.5 signature"); public_key .verify_signature(pkcs1v15_profile, message, &signature) - .unwrap(); + .expect("the generated public key must verify its PKCS#1 v1.5 signature"); public_key .verify_signature_with_scratch(pkcs1v15_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 verification must accept the matching signature"); key .sign_signature_with_scratch(pkcs1v15_profile, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 signing must succeed"); public_key .verify_signature(pkcs1v15_profile, message, &signature) - .unwrap(); + .expect("the generated public key must verify the scratch-backed PKCS#1 v1.5 signature"); public_key .verify_signature_with_scratch(pkcs1v15_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed verification must accept the scratch-backed PKCS#1 v1.5 signature"); let pss_profile = RsaSignatureProfile::pss(pss_profile); - key.sign_signature(pss_profile, message, &mut signature).unwrap(); - public_key.verify_signature(pss_profile, message, &signature).unwrap(); + key + .sign_signature(pss_profile, message, &mut signature) + .expect("the generated key must produce an RSA-PSS signature"); + public_key + .verify_signature(pss_profile, message, &signature) + .expect("the generated public key must verify its RSA-PSS signature"); public_key .verify_signature_with_scratch(pss_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed RSA-PSS verification must accept the matching signature"); key .sign_signature_with_scratch(pss_profile, message, &mut signature, &mut private_scratch) - .unwrap(); - public_key.verify_signature(pss_profile, message, &signature).unwrap(); + .expect("scratch-backed RSA-PSS signing must succeed"); + public_key + .verify_signature(pss_profile, message, &signature) + .expect("the generated public key must verify the scratch-backed RSA-PSS signature"); public_key .verify_signature_with_scratch(pss_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed verification must accept the scratch-backed RSA-PSS signature"); } let explicit_pss_profile = RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha384, 24); key .sign_signature_with_scratch(explicit_pss_profile, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("RSA-PSS signing with an explicit salt length must succeed"); public_key .verify_signature(explicit_pss_profile, message, &signature) - .unwrap(); + .expect("RSA-PSS verification must honor the explicit salt length"); public_key .verify_signature_with_scratch(explicit_pss_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed RSA-PSS verification must honor the explicit salt length"); key .sign_x509_signature_algorithm_der(X509_SHA256_WITH_RSA_ENCRYPTION, message, &mut signature) - .unwrap(); + .expect("the PKCS#1 v1.5 X.509 algorithm identifier must be accepted for signing"); x509_key .verify_signature_from_x509_algorithm_der(X509_SHA256_WITH_RSA_ENCRYPTION, message, &signature) - .unwrap(); + .expect("the X.509 key must verify the matching PKCS#1 v1.5 signature"); x509_key .verify_signature_from_x509_algorithm_der_with_scratch( X509_SHA256_WITH_RSA_ENCRYPTION, @@ -506,7 +545,7 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut x509_scratch, ) - .unwrap(); + .expect("scratch-backed X.509 verification must accept the matching PKCS#1 v1.5 signature"); key .sign_x509_signature_algorithm_der_with_scratch( X509_PSS_SHA256_ALGORITHM, @@ -514,10 +553,10 @@ fn root_surface_rsa_generated_key_end_to_end() { &mut signature, &mut private_scratch, ) - .unwrap(); + .expect("the RSA-PSS X.509 algorithm identifier must be accepted for scratch-backed signing"); x509_key .verify_signature_from_x509_algorithm_der(X509_PSS_SHA256_ALGORITHM, message, &signature) - .unwrap(); + .expect("the X.509 key must verify the matching RSA-PSS signature"); x509_key .verify_signature_from_x509_algorithm_der_with_scratch( X509_PSS_SHA256_ALGORITHM, @@ -525,17 +564,17 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut x509_scratch, ) - .unwrap(); + .expect("scratch-backed X.509 verification must accept the matching RSA-PSS signature"); let pss_sha256 = RsaSignatureProfile::pss(RsaPssProfile::Sha256); let pkcs1v15_sha256 = RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256); key .sign_tls13_signature_scheme(0x0804, message, &mut signature) - .unwrap(); + .expect("TLS 1.3 rsa_pss_rsae_sha256 signing must succeed"); x509_key .verify_expected_tls13_signature_scheme(0x0804, 0x0804, pss_sha256, message, &signature) - .unwrap(); + .expect("the matching TLS 1.3 RSA-PSS signature must verify"); x509_key .verify_expected_tls13_signature_scheme_with_scratch( 0x0804, @@ -545,13 +584,13 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut x509_scratch, ) - .unwrap(); + .expect("scratch-backed TLS 1.3 RSA-PSS verification must accept the matching signature"); key .sign_tls13_signature_scheme_with_scratch(0x0804, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("scratch-backed TLS 1.3 RSA-PSS signing must succeed"); x509_key .verify_expected_tls13_signature_scheme(0x0804, 0x0804, pss_sha256, message, &signature) - .unwrap(); + .expect("the scratch-backed TLS 1.3 RSA-PSS signature must verify"); x509_key .verify_expected_tls13_signature_scheme_with_scratch( 0x0804, @@ -561,14 +600,14 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut x509_scratch, ) - .unwrap(); + .expect("scratch-backed verification must accept the scratch-backed TLS 1.3 RSA-PSS signature"); key .sign_tls_certificate_signature_scheme(0x0401, message, &mut signature) - .unwrap(); + .expect("TLS certificate rsa_pkcs1_sha256 signing must succeed"); x509_key .verify_expected_tls_certificate_signature_scheme(0x0401, 0x0401, pkcs1v15_sha256, message, &signature) - .unwrap(); + .expect("the matching TLS certificate PKCS#1 v1.5 signature must verify"); x509_key .verify_expected_tls_certificate_signature_scheme_with_scratch( 0x0401, @@ -578,13 +617,13 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut x509_scratch, ) - .unwrap(); + .expect("scratch-backed TLS certificate verification must accept the matching signature"); key .sign_tls_certificate_signature_scheme_with_scratch(0x0401, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("scratch-backed TLS certificate PKCS#1 v1.5 signing must succeed"); x509_key .verify_expected_tls_certificate_signature_scheme(0x0401, 0x0401, pkcs1v15_sha256, message, &signature) - .unwrap(); + .expect("the scratch-backed TLS certificate signature must verify"); x509_key .verify_expected_tls_certificate_signature_scheme_with_scratch( 0x0401, @@ -594,17 +633,19 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut x509_scratch, ) - .unwrap(); + .expect("scratch-backed verification must accept the scratch-backed TLS certificate signature"); key .jwt_signer(RsaJwtAlgorithm::Ps256) .try_sign_into(message, &mut signature) - .unwrap(); + .expect("PS256 JWT signing must succeed"); let verifier = public_key.jwt_verifier(RsaJwtAlgorithm::Ps256); - verifier.verify("PS256", message, &signature).unwrap(); + verifier + .verify("PS256", message, &signature) + .expect("the matching PS256 JWT signature must verify"); verifier .verify_with_scratch("PS256", message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed PS256 JWT verification must accept the matching signature"); key .sign_signature_with_scratch( RsaJwtAlgorithm::Rs256.signature_profile(), @@ -612,26 +653,30 @@ fn root_surface_rsa_generated_key_end_to_end() { &mut signature, &mut private_scratch, ) - .unwrap(); + .expect("scratch-backed RS256 signing must succeed"); let verifier = public_key.jwt_verifier(RsaJwtAlgorithm::Rs256); - verifier.verify("RS256", message, &signature).unwrap(); + verifier + .verify("RS256", message, &signature) + .expect("the matching RS256 JWT signature must verify"); verifier .verify_with_scratch("RS256", message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed RS256 JWT verification must accept the matching signature"); - key.sign_cose_algorithm_id(-37, message, &mut signature).unwrap(); + key + .sign_cose_algorithm_id(-37, message, &mut signature) + .expect("COSE PS256 signing must succeed"); public_key .verify_expected_cose_algorithm_id(-37, -37, pss_sha256, message, &signature) - .unwrap(); + .expect("the matching COSE PS256 signature must verify"); public_key .verify_expected_cose_algorithm_id_with_scratch(-37, -37, pss_sha256, message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed COSE PS256 verification must accept the matching signature"); key .sign_cose_algorithm_id_with_scratch(-257, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("scratch-backed COSE RS256 signing must succeed"); public_key .verify_expected_cose_algorithm_id(-257, -257, pkcs1v15_sha256, message, &signature) - .unwrap(); + .expect("the matching COSE RS256 signature must verify"); public_key .verify_expected_cose_algorithm_id_with_scratch( -257, @@ -641,7 +686,7 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut public_scratch, ) - .unwrap(); + .expect("scratch-backed COSE RS256 verification must accept the matching signature"); let label = b"root-surface-rsa-label"; let plaintext = b"root-surface-rsa-oaep"; @@ -650,37 +695,41 @@ fn root_surface_rsa_generated_key_end_to_end() { for oaep_profile in [RsaOaepProfile::Sha256, RsaOaepProfile::Sha384, RsaOaepProfile::Sha512] { public_key .encrypt_oaep(oaep_profile, label, plaintext, &mut ciphertext) - .unwrap(); + .expect("OAEP encryption of the bounded plaintext must succeed"); public_key .encrypt_oaep_with_scratch(oaep_profile, label, plaintext, &mut ciphertext, &mut public_scratch) - .unwrap(); + .expect("scratch-backed OAEP encryption of the bounded plaintext must succeed"); let decrypted_len = key .decrypt_oaep(oaep_profile, label, &ciphertext, &mut decrypted) - .unwrap(); + .expect("OAEP decryption with the matching key and label must succeed"); assert_eq!(&decrypted[..decrypted_len], plaintext); let decrypted_len = key .decrypt_oaep_with_scratch(oaep_profile, label, &ciphertext, &mut decrypted, &mut private_scratch) - .unwrap(); + .expect("scratch-backed OAEP decryption with the matching key and label must succeed"); assert_eq!(&decrypted[..decrypted_len], plaintext); } let legacy_plaintext = b"root-surface-rsaes-pkcs1v15"; - public_key.encrypt_pkcs1v15(legacy_plaintext, &mut ciphertext).unwrap(); + public_key + .encrypt_pkcs1v15(legacy_plaintext, &mut ciphertext) + .expect("PKCS#1 v1.5 encryption of the bounded plaintext must succeed"); public_key .encrypt_pkcs1v15_with_scratch(legacy_plaintext, &mut ciphertext, &mut public_scratch) - .unwrap(); - let decrypted_len = key.decrypt_pkcs1v15(&ciphertext, &mut decrypted).unwrap(); + .expect("scratch-backed PKCS#1 v1.5 encryption of the bounded plaintext must succeed"); + let decrypted_len = key + .decrypt_pkcs1v15(&ciphertext, &mut decrypted) + .expect("PKCS#1 v1.5 decryption with the matching key must succeed"); assert_eq!(&decrypted[..decrypted_len], legacy_plaintext); let decrypted_len = key .decrypt_pkcs1v15_with_scratch(&ciphertext, &mut decrypted, &mut private_scratch) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 decryption with the matching key must succeed"); assert_eq!(&decrypted[..decrypted_len], legacy_plaintext); } #[test] #[cfg(all(feature = "rsa", feature = "getrandom"))] fn root_surface_rsa_default_generated_key_end_to_end() { - let key = RsaPrivateKey::generate(3072).unwrap(); + let key = RsaPrivateKey::generate(3072).expect("the default policy must generate a 3072-bit RSA key"); assert_eq!(key.public_key().modulus_bits(), 3072); let public_key = key.public_key(); let message = b"root-surface-rsa-default-generated-key"; @@ -690,19 +739,25 @@ fn root_surface_rsa_default_generated_key_end_to_end() { let pkcs1_der = key.to_pkcs1_der(); let pkcs8_der = key.to_pkcs8_der(); assert_eq!( - RsaPrivateKey::from_pkcs1_der(&pkcs1_der).unwrap().public_key(), + RsaPrivateKey::from_pkcs1_der(&pkcs1_der) + .expect("the default-policy private key must round-trip through PKCS#1 DER") + .public_key(), public_key ); assert_eq!( - RsaPrivateKey::from_pkcs8_der(&pkcs8_der).unwrap().public_key(), + RsaPrivateKey::from_pkcs8_der(&pkcs8_der) + .expect("the default-policy private key must round-trip through PKCS#8 DER") + .public_key(), public_key ); assert_eq!( - RsaPublicKey::from_pkcs1_der(&public_key.to_pkcs1_der()).unwrap(), + RsaPublicKey::from_pkcs1_der(&public_key.to_pkcs1_der()) + .expect("the default-policy public key must round-trip through PKCS#1 DER"), *public_key ); assert_eq!( - RsaPublicKey::from_spki_der(&public_key.to_spki_der()).unwrap(), + RsaPublicKey::from_spki_der(&public_key.to_spki_der()) + .expect("the default-policy public key must round-trip through SPKI DER"), *public_key ); @@ -710,18 +765,18 @@ fn root_surface_rsa_default_generated_key_end_to_end() { let pkcs1v15_profile = RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256); key .sign_signature_with_scratch(pkcs1v15_profile, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("default-policy scratch-backed PKCS#1 v1.5 signing must succeed"); public_key .verify_signature_with_scratch(pkcs1v15_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("default-policy scratch-backed PKCS#1 v1.5 verification must succeed"); let pss_profile = RsaSignatureProfile::pss(RsaPssProfile::Sha256); key .sign_signature_with_scratch(pss_profile, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("default-policy scratch-backed RSA-PSS signing must succeed"); public_key .verify_signature_with_scratch(pss_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("default-policy scratch-backed RSA-PSS verification must succeed"); let label = b"root-surface-rsa-default-label"; let plaintext = b"root-surface-rsa-default-oaep"; @@ -737,7 +792,7 @@ fn root_surface_rsa_default_generated_key_end_to_end() { &mut public_scratch, fill_rsa_random_from(&seed), ) - .unwrap(); + .expect("deterministic scratch-backed OAEP encryption must succeed"); let decrypted_len = key .decrypt_oaep_with_scratch( RsaOaepProfile::Sha256, @@ -746,7 +801,7 @@ fn root_surface_rsa_default_generated_key_end_to_end() { &mut decrypted, &mut private_scratch, ) - .unwrap(); + .expect("scratch-backed OAEP decryption of the deterministic ciphertext must succeed"); assert_eq!(&decrypted[..decrypted_len], plaintext); let legacy_plaintext = b"root-surface-rsa-default-rsaes-pkcs1v15"; @@ -757,10 +812,10 @@ fn root_surface_rsa_default_generated_key_end_to_end() { &mut public_scratch, fill_rsa_random_with(0x5b), ) - .unwrap(); + .expect("deterministic scratch-backed PKCS#1 v1.5 encryption must succeed"); let decrypted_len = key .decrypt_pkcs1v15_with_scratch(&ciphertext, &mut decrypted, &mut private_scratch) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 decryption of the deterministic ciphertext must succeed"); assert_eq!(&decrypted[..decrypted_len], legacy_plaintext); } @@ -771,13 +826,16 @@ fn root_surface_key_exchange_exports_compile() { let bob = X25519SecretKey::from_bytes([13u8; X25519SecretKey::LENGTH]); let alice_public: X25519PublicKey = (&alice).into(); let bob_public: X25519PublicKey = (&bob).into(); - let alice_shared = alice.diffie_hellman(&bob_public).unwrap(); - let bob_shared = X25519SharedSecret::diffie_hellman(&bob, &alice_public).unwrap(); + let alice_shared = alice + .diffie_hellman(&bob_public) + .expect("the fixed Alice and Bob X25519 keys must produce a shared secret"); + let bob_shared = X25519SharedSecret::diffie_hellman(&bob, &alice_public) + .expect("the fixed Bob and Alice X25519 keys must produce a shared secret"); assert_eq!(alice_public.as_bytes().len(), 32); assert_eq!(alice_shared.as_bytes().len(), 32); assert!(alice_shared.ct_eq(&bob_shared).declassify()); - let _ = X25519Error::new(); + let _error = X25519Error::new(); } #[test] @@ -801,6 +859,10 @@ fn root_surface_checksum_exports_compile() { assert_eq!(Crc32Ieee::checksum(data), Crc32::checksum(data)); assert_eq!(Crc32Castagnoli::checksum(data), Crc32C::checksum(data)); assert_eq!(Crc64Xz::checksum(data), Crc64::checksum(data)); + let _crc16_ccitt = Crc16Ccitt::checksum(data); + let _crc16_ibm = Crc16Ibm::checksum(data); + let _crc24_openpgp = Crc24OpenPgp::checksum(data); + let _crc64_nvme = Crc64Nvme::checksum(data); } #[test] @@ -831,38 +893,41 @@ fn root_surface_hash_exports_compile() { streaming.update(data); assert_eq!(oneshot, streaming.finalize()); - let mut xof = Blake3::xof(data); + let mut xof: Blake3XofReader = Blake3::xof(data); let mut out = [0u8; 16]; xof.squeeze(&mut out); - let mut shake = Shake256::xof(data); + let mut shake128: Shake128XofReader = Shake128::xof(data); + shake128.squeeze(&mut out); + + let mut shake: Shake256XofReader = Shake256::xof(data); shake.squeeze(&mut out); - let mut ascon = AsconXof::xof(data); + let mut ascon: AsconXofReader = AsconXof::xof(data); ascon.squeeze(&mut out); - let mut cshake = Cshake256::xof(b"", b"ctx=v1", data); + let mut cshake: Cshake256XofReader = Cshake256::xof(b"", b"ctx=v1", data); cshake.squeeze(&mut out); - let mut cshake128 = Cshake128::xof(b"", b"ctx=v1", data); + let mut cshake128: Cshake128XofReader = Cshake128::xof(b"", b"ctx=v1", data); cshake128.squeeze(&mut out); - let _: Option = None; - let mut cxof = AsconCxof128::xof(b"ctx=v1", data).unwrap(); + let mut cxof: AsconCxof128Reader = + AsconCxof128::xof(b"ctx=v1", data).expect("the short Ascon-CXOF customization must be accepted"); cxof.squeeze(&mut out); assert_eq!(Xxh3::hash(data), Xxh3_64::hash(data)); - let _ = RapidHash64::hash(data); + let _rapid_hash = RapidHash64::hash(data); let deterministic = RapidSeededState::new(42); - let mut collection = deterministic.build_hasher(); + let mut collection: RapidHasher = deterministic.build_hasher(); collection.write(data); - let _ = collection.finish(); + let _collection_hash = collection.finish(); let random = RapidRandomState::try_new_with(|seed| { seed.copy_from_slice(&42u64.to_le_bytes()); Ok::<_, ()>(()) }) - .unwrap(); - let _ = random.hash_one(data); + .expect("the deterministic seed filler must initialize RapidHash state"); + let _randomized_hash = random.hash_one(data); let mut stream = RapidStreamHasher::new(); stream.write(data); @@ -873,20 +938,22 @@ fn root_surface_hash_exports_compile() { #[test] #[cfg(all(feature = "hashes", feature = "std"))] fn digest_reader_writer_round_trip() { - use std::io::{Cursor, Read, Write}; + use std::io::Write; let data = b"hello digest reader writer"; let expected = Sha256::digest(data); // DigestReader: read data through and verify digest matches. - let mut reader = DigestReader::<_, Sha256>::new(Cursor::new(data.to_vec())); + let mut reader = DigestReader::<_, Sha256>::new(data.as_slice()); let mut sink = Vec::new(); - std::io::copy(&mut reader, &mut sink).unwrap(); + std::io::copy(&mut reader, &mut sink).expect("copying from an in-memory digest reader must succeed"); assert_eq!(reader.digest(), expected); // DigestWriter: write data through and verify digest matches. let mut writer = DigestWriter::<_, Sha256>::new(Vec::new()); - writer.write_all(data).unwrap(); + writer + .write_all(data) + .expect("writing to an in-memory digest writer must succeed"); let (out, digest) = writer.into_parts(); assert_eq!(&out, data); assert_eq!(digest, expected); @@ -897,12 +964,18 @@ fn digest_reader_writer_round_trip() { fn advanced_checksum_modules_compile() { fn assert_kernel_introspect() {} - let _: Crc32Config = Crc32::config(); - let _ = Crc32Force::Auto; - let _ = DispatchInfo::current(); - let _ = kernel_for::(64); - let _ = is_hardware_accelerated(); - let _ = rscrypto::platform::describe(); + let _crc16_config: Crc16Config = Crc16Ccitt::config(); + let _crc24_config: Crc24Config = Crc24OpenPgp::config(); + let _crc32_config: Crc32Config = Crc32::config(); + let _crc64_config: Crc64Config = Crc64::config(); + let _crc16_force = Crc16Force::Auto; + let _crc24_force = Crc24Force::Auto; + let _crc32_force = Crc32Force::Auto; + let _crc64_force = Crc64Force::Auto; + let _dispatch = DispatchInfo::current(); + let _kernel = kernel_for::(64); + let _accelerated = is_hardware_accelerated(); + let _platform = rscrypto::platform::describe(); assert_kernel_introspect::(); } @@ -911,14 +984,14 @@ fn advanced_checksum_modules_compile() { fn advanced_hash_modules_compile() { fn assert_hash_kernel_introspect() {} - let _ = HashDispatchInfo::current(); - let _ = hash_kernel_for::(64); - let _ = hash_kernel_for::(64); - let _ = hash_kernel_for::(64); - let _ = hash_kernel_for::(64); - let _ = hash_kernel_for::(64); - let _ = hash_kernel_for::(64); - let _ = hash_kernel_for::(64); + let _dispatch = HashDispatchInfo::current(); + let _sha256_kernel = hash_kernel_for::(64); + let _shake256_kernel = hash_kernel_for::(64); + let _blake3_kernel = hash_kernel_for::(64); + let _ascon_hash_kernel = hash_kernel_for::(64); + let _ascon_xof_kernel = hash_kernel_for::(64); + let _ascon_cxof_kernel = hash_kernel_for::(64); + let _xxh3_kernel = hash_kernel_for::(64); assert_hash_kernel_introspect::(); assert_hash_kernel_introspect::(); diff --git a/tests/rsa_allocations.rs b/tests/rsa_allocations.rs index 1e9d7369..f45d0edf 100644 --- a/tests/rsa_allocations.rs +++ b/tests/rsa_allocations.rs @@ -8,8 +8,8 @@ use std::alloc::System; use rsa::{BigUint, RsaPrivateKey as RustCryptoRsaPrivateKey, pkcs1::EncodeRsaPrivateKey}; use rscrypto::{ - RsaEncryptionError, RsaJwtAlgorithm, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKey, - RsaPublicKeyPolicy, RsaSignatureProfile, RsaX509PublicKey, + RsaBlindingPair, RsaEncryptionError, RsaJwtAlgorithm, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, + RsaPssProfile, RsaPublicKey, RsaPublicKeyPolicy, RsaSignatureProfile, RsaX509PublicKey, }; static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0); @@ -74,41 +74,49 @@ const X509_PSS_DEFAULT_SHA1_ALGORITHM: &[u8] = &[ ]; fn hex_to_vec(hex: &str) -> Vec { - assert_eq!(hex.len() % 2, 0); - let mut out = Vec::with_capacity(hex.len() / 2); - for chunk in hex.as_bytes().chunks_exact(2) { - out.push((hex_value(chunk[0]) << 4) | hex_value(chunk[1])); + let (chunks, remainder) = hex.as_bytes().as_chunks::<2>(); + assert!(remainder.is_empty(), "hex input must contain complete byte pairs"); + let mut out = Vec::with_capacity(chunks.len()); + for &[hi, lo] in chunks { + let hi = hex_value(hi).expect("hex fixtures must contain only ASCII hexadecimal digits"); + let lo = hex_value(lo).expect("hex fixtures must contain only ASCII hexadecimal digits"); + out.push((hi << 4) | lo); } out } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("a short DER length must fit in one byte")]; } let bytes = len.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .expect("a DER length of at least 128 must contain a nonzero byte"); let len_bytes = &bytes[first_nonzero..]; - let mut out = Vec::with_capacity(1 + len_bytes.len()); - out.push(0x80 | len_bytes.len() as u8); + let mut out = Vec::with_capacity(1usize.strict_add(len_bytes.len())); + out.push(0x80 | u8::try_from(len_bytes.len()).expect("a usize DER length prefix must fit in one byte")); out.extend_from_slice(len_bytes); out } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + der_len(value.len()).len() + value.len()); + let encoded_len = der_len(value.len()); + let capacity = 1usize.strict_add(encoded_len.len()).strict_add(value.len()); + let mut out = Vec::with_capacity(capacity); out.push(tag); - out.extend_from_slice(&der_len(value.len())); + out.extend_from_slice(&encoded_len); out.extend_from_slice(value); out } @@ -148,7 +156,7 @@ fn bit_string(value: &[u8]) -> Vec { fn integer_unsigned(value: &[u8]) -> Vec { let first_nonzero = value.iter().position(|&byte| byte != 0); let value = first_nonzero.map_or(&[0u8][..], |index| &value[index..]); - let mut encoded = Vec::with_capacity(value.len() + usize::from(value[0] & 0x80 != 0)); + let mut encoded = Vec::with_capacity(value.len().strict_add(usize::from(value[0] & 0x80 != 0))); if value[0] & 0x80 != 0 { encoded.push(0); } @@ -167,7 +175,10 @@ fn algorithm_identifier(algorithm_oid: &[u8], params: Option<&[u8]>) -> Vec fn exponent_bytes(value: u64) -> Vec { let bytes = value.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap_or(bytes.len() - 1); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .unwrap_or_else(|| bytes.len().strict_sub(1)); bytes[first_nonzero..].to_vec() } @@ -179,7 +190,7 @@ fn pkcs1_from_public_key(key: &RsaPublicKey) -> Vec { } fn spki_for_pkcs1_with_algorithm(pkcs1: &[u8], algorithm: &[u8]) -> Vec { - let mut subject_public_key = Vec::with_capacity(1 + pkcs1.len()); + let mut subject_public_key = Vec::with_capacity(1usize.strict_add(pkcs1.len())); subject_public_key.push(0); subject_public_key.extend_from_slice(pkcs1); @@ -196,11 +207,13 @@ fn pss_algorithm_spki_from_rsa_encryption_spki(spki: &[u8]) -> Vec { } fn legacy_public_key_from_spki(spki: &[u8]) -> RsaPublicKey { - RsaPublicKey::from_spki_der_with_policy(spki, &RsaPublicKeyPolicy::legacy_verification()).unwrap() + RsaPublicKey::from_spki_der_with_policy(spki, &RsaPublicKeyPolicy::legacy_verification()) + .expect("the SPKI fixture must satisfy the legacy verification policy") } fn legacy_x509_public_key_from_spki(spki: &[u8]) -> RsaX509PublicKey { - RsaX509PublicKey::from_spki_der_with_policy(spki, &RsaPublicKeyPolicy::legacy_verification()).unwrap() + RsaX509PublicKey::from_spki_der_with_policy(spki, &RsaPublicKeyPolicy::legacy_verification()) + .expect("the X.509 SPKI fixture must satisfy the legacy verification policy") } fn minimal_tbs_certificate(signature_algorithm_der: &[u8]) -> Vec { @@ -211,7 +224,7 @@ fn minimal_tbs_certificate(signature_algorithm_der: &[u8]) -> Vec { } fn x509_certificate(tbs_certificate_der: &[u8], signature_algorithm_der: &[u8], signature: &[u8]) -> Vec { - let mut signature_value = Vec::with_capacity(signature.len() + 1); + let mut signature_value = Vec::with_capacity(signature.len().strict_add(1)); signature_value.push(0); signature_value.extend_from_slice(signature); @@ -282,32 +295,35 @@ fn rustcrypto_fixture_private_key() -> RustCryptoRsaPrivateKey { b"00d397b84d98a4c26138ed1b695a8106ead91d553bf06041b62d3fdc50a041e222b8f4529689c1b82c5e71554f5dd69fa2f4b6158cf0dbeb57811a0fc327e1f28e74fe74d3bc166c1eabdc1b8b57b934ca8be5b00b4f29975bcc99acaf415b59bb28a6782bb41a2c3c2976b3c18dbadef62f00c6bb226640095096c0cc60d22fe7ef987d75c6a81b10d96bf292028af110dc7cc1bbc43d22adab379a0cd5d8078cc780ff5cd6209dea34c922cf784f7717e428d75b5aec8ff30e5f0141510766e2e0ab8d473c84e8710b2b98227c3db095337ad3452f19e2b9bfbccdd8148abf6776fa552775e6e75956e45229ae5a9c46949bab1e622f0e48f56524a84ed3483b", 16, ) - .unwrap(), - BigUint::parse_bytes(b"010001", 16).unwrap(), + .expect("the fixed RSA modulus must be valid hexadecimal"), + BigUint::parse_bytes(b"010001", 16).expect("the fixed RSA exponent must be valid hexadecimal"), BigUint::parse_bytes( b"00c4e70c689162c94c660828191b52b4d8392115df486a9adbe831e458d73958320dc1b755456e93701e9702d76fb0b92f90e01d1fe248153281fe79aa9763a92fae69d8d7ecd144de29fa135bd14f9573e349e45031e3b76982f583003826c552e89a397c1a06bd2163488630d92e8c2bb643d7abef700da95d685c941489a46f54b5316f62b5d2c3a7f1bbd134cb37353a44683fdc9d95d36458de22f6c44057fe74a0a436c4308f73f4da42f35c47ac16a7138d483afc91e41dc3a1127382e0c0f5119b0221b4fc639d6b9c38177a6de9b526ebd88c38d7982c07f98a0efd877d508aae275b946915c02e2e1106d175d74ec6777f5e80d12c053d9c7be1e341", 16, ) - .unwrap(), + .expect("the fixed RSA private exponent must be valid hexadecimal"), vec![ BigUint::parse_bytes( b"00f827bbf3a41877c7cc59aebf42ed4b29c32defcb8ed96863d5b090a05a8930dd624a21c9dcf9838568fdfa0df65b8462a5f2ac913d6c56f975532bd8e78fb07bd405ca99a484bcf59f019bbddcb3933f2bce706300b4f7b110120c5df9018159067c35da3061a56c8635a52b54273b31271b4311f0795df6021e6355e1a42e61", 16, ) - .unwrap(), + .expect("the fixed first RSA prime must be valid hexadecimal"), BigUint::parse_bytes( b"00da4817ce0089dd36f2ade6a3ff410c73ec34bf1b4f6bda38431bfede11cef1f7f6efa70e5f8063a3b1f6e17296ffb15feefa0912a0325b8d1fd65a559e717b5b961ec345072e0ec5203d03441d29af4d64054a04507410cf1da78e7b6119d909ec66e6ad625bf995b279a4b3c5be7d895cd7c5b9c4c497fde730916fcdb4e41b", 16, ) - .unwrap(), + .expect("the fixed second RSA prime must be valid hexadecimal"), ], ) - .unwrap() + .expect("the fixed RSA components must form a valid private key") } fn private_key() -> RsaPrivateKey { - let der = rustcrypto_fixture_private_key().to_pkcs1_der().unwrap(); - RsaPrivateKey::from_pkcs1_der_with_policy(der.as_bytes(), &RsaPublicKeyPolicy::legacy_verification()).unwrap() + let der = rustcrypto_fixture_private_key() + .to_pkcs1_der() + .expect("the RustCrypto private-key fixture must encode as PKCS#1"); + RsaPrivateKey::from_pkcs1_der_with_policy(der.as_bytes(), &RsaPublicKeyPolicy::legacy_verification()) + .expect("rscrypto must decode the RustCrypto private-key fixture") } fn factor_two_and_inverse(modulus: &[u8]) -> (Vec, Vec) { @@ -319,32 +335,32 @@ fn factor_two_and_inverse(modulus: &[u8]) -> (Vec, Vec) { let mut plus_one = modulus.to_vec(); let mut carry = 1u16; for byte in plus_one.iter_mut().rev() { - let sum = u16::from(*byte) + carry; - *byte = sum as u8; + let sum = u16::from(*byte).strict_add(carry); + *byte = sum.to_le_bytes()[0]; carry = sum >> 8; if carry == 0 { break; } } if carry != 0 { - plus_one.insert(0, carry as u8); + plus_one.insert(0, u8::try_from(carry).expect("addition carry must be at most one")); } let mut quotient = Vec::with_capacity(plus_one.len()); let mut remainder = 0u16; for byte in plus_one { let value = (remainder << 8) | u16::from(byte); - quotient.push((value / 2) as u8); + quotient.push(u8::try_from(value / 2).expect("a base-256 long-division digit must fit in one byte")); remainder = value % 2; } let first_nonzero = quotient .iter() .position(|&byte| byte != 0) - .unwrap_or(quotient.len() - 1); + .unwrap_or_else(|| quotient.len().strict_sub(1)); let inverse = quotient[first_nonzero..].to_vec(); let mut inverse_fixed = vec![0u8; modulus.len()]; - inverse_fixed[modulus.len() - inverse.len()..].copy_from_slice(&inverse); + inverse_fixed[modulus.len().strict_sub(inverse.len())..].copy_from_slice(&inverse); (factor, inverse_fixed) } @@ -379,12 +395,12 @@ fn reused_scratch_rsa_operations_do_not_allocate() { key .public_operation_with_scratch(&input, &mut out, &mut scratch) - .unwrap(); + .expect("the warm-up RSA public operation must succeed"); reset_allocations(); key .public_operation_with_scratch(&input, &mut out, &mut scratch) - .unwrap(); + .expect("the measured RSA public operation must succeed"); assert_eq!(allocation_count(), 0); let sig = pss_signature_sha256(); @@ -392,7 +408,7 @@ fn reused_scratch_rsa_operations_do_not_allocate() { reset_allocations(); key .verify_pss_with_scratch(RsaPssProfile::Sha256, MESSAGE_PSS, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed PSS verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); @@ -403,20 +419,20 @@ fn reused_scratch_rsa_operations_do_not_allocate() { &sig, &mut scratch, ) - .unwrap(); + .expect("typed scratch-backed PSS verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); key .jwt_verifier(RsaJwtAlgorithm::Ps256) .verify_with_scratch("PS256", MESSAGE_PSS, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed PS256 verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); key .verify_expected_cose_algorithm_id_with_scratch(-37, -37, pss_sha256, MESSAGE_PSS, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed PS256 COSE verification must succeed"); assert_eq!(allocation_count(), 0); let seed = [0x42; 32]; @@ -430,7 +446,7 @@ fn reused_scratch_rsa_operations_do_not_allocate() { &mut scratch, fill_rsa_random_from(&seed), ) - .unwrap(); + .expect("scratch-backed OAEP encryption must succeed"); assert_eq!(allocation_count(), 0); let x509_key = legacy_x509_public_key_from_spki(&pss_spki()); @@ -438,13 +454,13 @@ fn reused_scratch_rsa_operations_do_not_allocate() { reset_allocations(); x509_key .verify_signature_from_x509_algorithm_der_with_scratch(X509_PSS_SHA256_ALGORITHM, MESSAGE_PSS, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed X.509 PSS verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); x509_key .verify_expected_tls13_signature_scheme_with_scratch(0x0804, 0x0804, pss_sha256, MESSAGE_PSS, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed TLS 1.3 PSS verification must succeed"); assert_eq!(allocation_count(), 0); let key = legacy_public_key_from_spki(&pkcs1v15_spki()); @@ -454,7 +470,7 @@ fn reused_scratch_rsa_operations_do_not_allocate() { reset_allocations(); key .verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, MESSAGE_PKCS1V15, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); @@ -465,20 +481,20 @@ fn reused_scratch_rsa_operations_do_not_allocate() { &sig, &mut scratch, ) - .unwrap(); + .expect("typed scratch-backed PKCS#1 v1.5 verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); key .jwt_verifier(RsaJwtAlgorithm::Rs256) .verify_with_scratch("RS256", MESSAGE_PKCS1V15, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed RS256 verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); key .verify_expected_cose_algorithm_id_with_scratch(-257, -257, pkcs1v15_sha256, MESSAGE_PKCS1V15, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed RS256 COSE verification must succeed"); assert_eq!(allocation_count(), 0); let message = b"scratch-backed RSAES-PKCS1-v1_5 encryption"; @@ -486,7 +502,7 @@ fn reused_scratch_rsa_operations_do_not_allocate() { reset_allocations(); key .encrypt_pkcs1v15_with_random_fill_and_scratch(message, &mut ciphertext, &mut scratch, fill_rsa_random_with(0x5d)) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 encryption must succeed"); assert_eq!(allocation_count(), 0); let x509_key = legacy_x509_public_key_from_spki(&pkcs1v15_spki()); @@ -499,7 +515,7 @@ fn reused_scratch_rsa_operations_do_not_allocate() { &sig, &mut scratch, ) - .unwrap(); + .expect("scratch-backed X.509 PKCS#1 v1.5 verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); @@ -512,7 +528,7 @@ fn reused_scratch_rsa_operations_do_not_allocate() { &sig, &mut scratch, ) - .unwrap(); + .expect("scratch-backed TLS certificate PKCS#1 v1.5 verification must succeed"); assert_eq!(allocation_count(), 0); assert_one_shot_protocol_rejects_fail_before_scratch_allocation(); @@ -651,12 +667,11 @@ fn assert_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { .sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, b"private scratch allocation PKCS1v15", - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, &mut scratch, ) - .unwrap(); + .expect("scratch-backed blinded PKCS#1 v1.5 signing must succeed"); assert_eq!(allocation_count(), 0); let salt = [0x7a; 32]; @@ -666,12 +681,11 @@ fn assert_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { RsaPssProfile::Sha256, b"private scratch allocation PSS", &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, &mut scratch, ) - .unwrap(); + .expect("scratch-backed blinded PSS signing must succeed"); assert_eq!(allocation_count(), 0); let label = b"private-scratch-allocation"; @@ -687,7 +701,7 @@ fn assert_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { &mut ciphertext, fill_rsa_random_from(&seed), ) - .unwrap(); + .expect("OAEP fixture encryption must succeed"); reset_allocations(); let len = key @@ -695,12 +709,11 @@ fn assert_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, &mut scratch, ) - .unwrap(); + .expect("scratch-backed blinded OAEP decryption must succeed"); assert_eq!(&decrypted[..len], plaintext); assert_eq!(allocation_count(), 0); @@ -708,18 +721,17 @@ fn assert_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { key .public_key() .encrypt_pkcs1v15_with_random_fill(pkcs1v15_plaintext, &mut ciphertext, fill_rsa_random_with(0x5d)) - .unwrap(); + .expect("PKCS#1 v1.5 fixture encryption must succeed"); reset_allocations(); let len = key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, &mut scratch, ) - .unwrap(); + .expect("scratch-backed blinded PKCS#1 v1.5 decryption must succeed"); assert_eq!(&decrypted[..len], pkcs1v15_plaintext); assert_eq!(allocation_count(), 0); } @@ -738,7 +750,7 @@ fn assert_rng_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { &mut signature, &mut scratch, ) - .unwrap(); + .expect("scratch-backed randomized PKCS#1 v1.5 signing must succeed"); assert_eq!(allocation_count(), 0); key .public_key() @@ -747,7 +759,7 @@ fn assert_rng_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { b"rng private scratch allocation PKCS1v15", &signature, ) - .unwrap(); + .expect("the randomized PKCS#1 v1.5 signature must verify"); reset_allocations(); key @@ -757,12 +769,12 @@ fn assert_rng_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { &mut signature, &mut scratch, ) - .unwrap(); + .expect("scratch-backed randomized PSS signing must succeed"); assert_eq!(allocation_count(), 0); key .public_key() .verify_pss(RsaPssProfile::Sha256, b"rng private scratch allocation PSS", &signature) - .unwrap(); + .expect("the randomized PSS signature must verify"); let label = b"rng-private-scratch-allocation"; let plaintext = b"rng private scratch allocation OAEP"; @@ -777,12 +789,12 @@ fn assert_rng_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { &mut ciphertext, fill_rsa_random_from(&seed), ) - .unwrap(); + .expect("OAEP fixture encryption must succeed"); reset_allocations(); let len = key .decrypt_oaep_with_scratch(RsaOaepProfile::Sha256, label, &ciphertext, &mut decrypted, &mut scratch) - .unwrap(); + .expect("scratch-backed randomized OAEP decryption must succeed"); assert_eq!(&decrypted[..len], plaintext); assert_eq!(allocation_count(), 0); @@ -790,12 +802,12 @@ fn assert_rng_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { key .public_key() .encrypt_pkcs1v15_with_random_fill(pkcs1v15_plaintext, &mut ciphertext, fill_rsa_random_with(0x5d)) - .unwrap(); + .expect("PKCS#1 v1.5 fixture encryption must succeed"); reset_allocations(); let len = key .decrypt_pkcs1v15_with_scratch(&ciphertext, &mut decrypted, &mut scratch) - .unwrap(); + .expect("scratch-backed randomized PKCS#1 v1.5 decryption must succeed"); assert_eq!(&decrypted[..len], pkcs1v15_plaintext); assert_eq!(allocation_count(), 0); } diff --git a/tests/rsa_leakage.rs b/tests/rsa_leakage.rs index 59db0371..2f86cb87 100644 --- a/tests/rsa_leakage.rs +++ b/tests/rsa_leakage.rs @@ -4,7 +4,7 @@ use core::hint::black_box; use std::time::Instant; use rscrypto::{ - RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKeyPolicy, + RsaBlindingPair, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKeyPolicy, auth::rsa::diag_rsa_blinding_factor_inverse, }; use serde_json::Value; @@ -48,7 +48,7 @@ struct OnlineStats { impl OnlineStats { fn push(&mut self, value: f64) { - self.n += 1; + self.n = self.n.strict_add(1); let delta = value - self.mean; self.mean += delta / self.n as f64; let delta2 = value - self.mean; @@ -56,7 +56,11 @@ impl OnlineStats { } fn variance(&self) -> f64 { - if self.n > 1 { self.m2 / (self.n - 1) as f64 } else { 0.0 } + if self.n > 1 { + self.m2 / self.n.strict_sub(1) as f64 + } else { + 0.0 + } } } @@ -114,13 +118,13 @@ where }; let index = if use_random { let index = random_index; - random_index += 1; - random_remaining -= 1; + random_index = random_index.strict_add(1); + random_remaining = random_remaining.strict_sub(1); index } else { let index = fixed_index; - fixed_index += 1; - fixed_remaining -= 1; + fixed_index = fixed_index.strict_add(1); + fixed_remaining = fixed_remaining.strict_sub(1); index }; @@ -150,17 +154,19 @@ fn hex_to_vec(hex: &str) -> Vec { assert_eq!(hex.len() % 2, 0); let mut out = Vec::with_capacity(hex.len() / 2); for chunk in hex.as_bytes().chunks_exact(2) { - out.push((hex_value(chunk[0]) << 4) | hex_value(chunk[1])); + let high = hex_value(chunk[0]).expect("leakage fixture must contain hexadecimal digits"); + let low = hex_value(chunk[1]).expect("leakage fixture must contain hexadecimal digits"); + out.push((high << 4) | low); } out } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } @@ -192,32 +198,32 @@ fn factor_two_and_inverse(modulus: &[u8]) -> (Vec, Vec) { let mut plus_one = modulus.to_vec(); let mut carry = 1u16; for byte in plus_one.iter_mut().rev() { - let sum = u16::from(*byte) + carry; - *byte = sum as u8; + let sum = u16::from(*byte).strict_add(carry); + *byte = u8::try_from(sum & 0xff).expect("masked addition result must fit u8"); carry = sum >> 8; if carry == 0 { break; } } if carry != 0 { - plus_one.insert(0, carry as u8); + plus_one.insert(0, u8::try_from(carry).expect("addition carry must fit u8")); } let mut quotient = Vec::with_capacity(plus_one.len()); let mut remainder = 0u16; for byte in plus_one { let value = (remainder << 8) | u16::from(byte); - quotient.push((value / 2) as u8); + quotient.push(u8::try_from(value / 2).expect("base-256 division quotient must fit u8")); remainder = value % 2; } let first_nonzero = quotient .iter() .position(|&byte| byte != 0) - .unwrap_or(quotient.len() - 1); + .unwrap_or_else(|| quotient.len().strict_sub(1)); let inverse = "ient[first_nonzero..]; let mut inverse_fixed = vec![0u8; modulus.len()]; - inverse_fixed[modulus.len() - inverse.len()..].copy_from_slice(inverse); + inverse_fixed[modulus.len().strict_sub(inverse.len())..].copy_from_slice(inverse); (factor, inverse_fixed) } @@ -274,7 +280,7 @@ fn fixed_and_random_oaep_ciphertexts(key: &RsaPrivateKey, count: usize) -> (Vec< fn fixed_and_random_pkcs1v15_ciphertexts(key: &RsaPrivateKey, count: usize) -> (Vec, Vec>) { let len = key.public_key().modulus().len(); let plaintext_len = b"fixed pkcs1v15 leakage text".len(); - let padding_len = len - plaintext_len - 3; + let padding_len = len.strict_sub(plaintext_len).strict_sub(3); let fixed_seed = vec![0x5a; padding_len]; let mut fixed = vec![0u8; len]; key @@ -288,7 +294,7 @@ fn fixed_and_random_pkcs1v15_ciphertexts(key: &RsaPrivateKey, count: usize) -> ( let mut seed = vec![0u8; padding_len]; for byte in &mut seed { while *byte == 0 { - *byte = rng.next_u64() as u8; + *byte = u8::try_from(rng.next_u64() & 0xff).expect("masked PRNG output must fit u8"); } } let mut plaintext = vec![0u8; plaintext_len]; @@ -361,8 +367,7 @@ fn rsa_private_operations_do_not_show_first_order_timing_leakage() { .sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, black_box(message), - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), black_box(&mut out), &mut scratch, ) @@ -389,8 +394,7 @@ fn rsa_private_operations_do_not_show_first_order_timing_leakage() { RsaPssProfile::Sha256, black_box(message), &pss_salt, - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), black_box(&mut out), &mut scratch, ) @@ -417,8 +421,7 @@ fn rsa_private_operations_do_not_show_first_order_timing_leakage() { RsaOaepProfile::Sha256, b"leakage-label", black_box(ciphertext), - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), black_box(&mut out), &mut scratch, ) @@ -443,8 +446,7 @@ fn rsa_private_operations_do_not_show_first_order_timing_leakage() { let plaintext_len = key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( black_box(ciphertext), - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), black_box(&mut out), &mut scratch, ) diff --git a/tests/rsa_nist_cavp.rs b/tests/rsa_nist_cavp.rs index b5ed390b..afe96076 100644 --- a/tests/rsa_nist_cavp.rs +++ b/tests/rsa_nist_cavp.rs @@ -5,8 +5,8 @@ extern crate alloc; use alloc::collections::BTreeMap; use rscrypto::{ - RsaKeyError, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPssProfile, RsaPublicKey, RsaPublicKeyPolicy, - RsaSignatureProfile, + RsaBlindingPair, RsaKeyError, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPssProfile, RsaPublicKey, + RsaPublicKeyPolicy, RsaSignatureProfile, }; use serde_json::Value; @@ -19,24 +19,41 @@ struct Counts { invalid: usize, } +#[derive(Clone, Copy)] +enum SigGenScheme { + Pkcs1v15, + Pss, +} + +#[derive(Clone, Copy)] +enum CavpResult { + Pass, + Fail, +} + fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("short DER length must fit in one byte")]; } let bytes = len.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .expect("long DER length must contain a non-zero byte"); let len_bytes = &bytes[first_nonzero..]; - let mut out = Vec::with_capacity(1 + len_bytes.len()); - out.push(0x80 | len_bytes.len() as u8); + let mut out = Vec::with_capacity(1usize.strict_add(len_bytes.len())); + out.push(0x80 | u8::try_from(len_bytes.len()).expect("DER length-of-length must fit in one byte")); out.extend_from_slice(len_bytes); out } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + der_len(value.len()).len() + value.len()); + let encoded_len = der_len(value.len()); + let capacity = 1usize.strict_add(encoded_len.len()).strict_add(value.len()); + let mut out = Vec::with_capacity(capacity); out.push(tag); - out.extend_from_slice(&der_len(value.len())); + out.extend_from_slice(&encoded_len); out.extend_from_slice(value); out } @@ -44,7 +61,7 @@ fn tlv(tag: u8, value: &[u8]) -> Vec { fn integer_unsigned(value: &[u8]) -> Vec { let first_nonzero = value.iter().position(|&byte| byte != 0); let value = first_nonzero.map_or(&[0u8][..], |index| &value[index..]); - let mut encoded = Vec::with_capacity(value.len() + usize::from(value[0] & 0x80 != 0)); + let mut encoded = Vec::with_capacity(value.len().strict_add(usize::from(value[0] & 0x80 != 0))); if value[0] & 0x80 != 0 { encoded.push(0); } @@ -72,7 +89,9 @@ fn hex_to_vec(hex: &str) -> Vec { let mut out = Vec::with_capacity(hex.len() / 2); for chunk in hex.as_bytes().chunks_exact(2) { - out.push((hex_value(chunk[0]) << 4) | hex_value(chunk[1])); + let high = hex_value(chunk[0]).expect("CAVP fixture must contain hexadecimal digits"); + let low = hex_value(chunk[1]).expect("CAVP fixture must contain hexadecimal digits"); + out.push((high << 4) | low); } out } @@ -82,7 +101,7 @@ fn hex_to_canonical_vec(hex: &str) -> Vec { let first_nonzero = value .iter() .position(|&byte| byte != 0) - .unwrap_or(value.len().strict_sub(1)); + .unwrap_or_else(|| value.len().strict_sub(1)); if first_nonzero != 0 { value.drain(..first_nonzero); } @@ -99,44 +118,67 @@ fn hex_to_u64(hex: &str) -> u64 { value } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } fn field<'a>(value: &'a Value, name: &'static str) -> &'a str { value[name] .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + .expect("CAVP fixture must contain the requested string field") } fn pkcs1_profile(sha: &str) -> RsaPkcs1v15Profile { match sha { - "SHA256" => RsaPkcs1v15Profile::Sha256, - "SHA384" => RsaPkcs1v15Profile::Sha384, - "SHA512" => RsaPkcs1v15Profile::Sha512, - other => panic!("unsupported CAVP PKCS1v1.5 hash `{other}`"), + "SHA256" => Some(RsaPkcs1v15Profile::Sha256), + "SHA384" => Some(RsaPkcs1v15Profile::Sha384), + "SHA512" => Some(RsaPkcs1v15Profile::Sha512), + _ => None, } + .expect("CAVP fixture must use a supported PKCS#1 v1.5 hash") } fn pss_profile(sha: &str) -> RsaPssProfile { match sha { - "SHA256" => RsaPssProfile::Sha256, - "SHA384" => RsaPssProfile::Sha384, - "SHA512" => RsaPssProfile::Sha512, - other => panic!("unsupported CAVP PSS hash `{other}`"), + "SHA256" => Some(RsaPssProfile::Sha256), + "SHA384" => Some(RsaPssProfile::Sha384), + "SHA512" => Some(RsaPssProfile::Sha512), + _ => None, } + .expect("CAVP fixture must use a supported PSS hash") } -fn signature_profile(scheme: &str, sha: &str, salt_len: Option) -> RsaSignatureProfile { +fn siggen_scheme(scheme: &str) -> SigGenScheme { match scheme { - "pkcs1v15" => RsaSignatureProfile::pkcs1v15(pkcs1_profile(sha)), - "pss" => RsaSignatureProfile::pss_with_salt_len(pss_profile(sha), salt_len.unwrap() as usize), - other => panic!("unsupported CAVP RSA signature scheme `{other}`"), + "pkcs1v15" => Some(SigGenScheme::Pkcs1v15), + "pss" => Some(SigGenScheme::Pss), + _ => None, + } + .expect("CAVP fixture must use a supported RSA signature scheme") +} + +fn cavp_result(result: &str) -> CavpResult { + match result { + "P" => Some(CavpResult::Pass), + "F" => Some(CavpResult::Fail), + _ => None, + } + .expect("CAVP fixture result must be P or F") +} + +fn signature_profile(scheme: &str, sha: &str, salt_len: Option) -> RsaSignatureProfile { + match siggen_scheme(scheme) { + SigGenScheme::Pkcs1v15 => RsaSignatureProfile::pkcs1v15(pkcs1_profile(sha)), + SigGenScheme::Pss => RsaSignatureProfile::pss_with_salt_len( + pss_profile(sha), + usize::try_from(salt_len.expect("CAVP PSS case must provide a salt length")) + .expect("CAVP PSS salt length must fit usize"), + ), } } @@ -187,7 +229,8 @@ fn nist_cavp_odd_public_exponents_require_explicit_policy() { ); let policy = RsaPublicKeyPolicy::legacy_verification().allow_legacy_odd_exponents(); - let key = RsaPublicKey::from_pkcs1_der_with_policy(&key_der, &policy).unwrap(); + let key = RsaPublicKey::from_pkcs1_der_with_policy(&key_der, &policy) + .expect("CAVP odd-exponent RSA key must parse under the explicit legacy policy"); assert_eq!(key.modulus_bits(), 2048); } @@ -242,15 +285,14 @@ fn nist_cavp_sha2_siggen_private_operations_match_expected_signatures() { let blinding_factor = fixed_width_one(key.public_key().modulus().len()); let blinding_factor_inverse = fixed_width_one(key.public_key().modulus().len()); - match field(test, "scheme") { - "pkcs1v15" => { + match siggen_scheme(field(test, "scheme")) { + SigGenScheme::Pkcs1v15 => { pkcs1v15 = pkcs1v15.strict_add(1); key .sign_pkcs1v15_with_blinding_factor( pkcs1_profile(field(test, "sha")), &message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, ) .expect("CAVP PKCS1v1.5 private signing must succeed"); @@ -259,8 +301,7 @@ fn nist_cavp_sha2_siggen_private_operations_match_expected_signatures() { .sign_pkcs1v15_with_blinding_factor_and_scratch( pkcs1_profile(field(test, "sha")), &message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_signature, &mut scratch, ) @@ -274,7 +315,7 @@ fn nist_cavp_sha2_siggen_private_operations_match_expected_signatures() { .verify_pkcs1v15(pkcs1_profile(field(test, "sha")), &message, &signature) .expect("CAVP PKCS1v1.5 generated signature must verify"); } - "pss" => { + SigGenScheme::Pss => { pss = pss.strict_add(1); let salt = hex_to_vec(field(test, "salt")); key @@ -282,8 +323,7 @@ fn nist_cavp_sha2_siggen_private_operations_match_expected_signatures() { pss_profile(field(test, "sha")), &message, &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, ) .expect("CAVP PSS private signing must succeed"); @@ -293,8 +333,7 @@ fn nist_cavp_sha2_siggen_private_operations_match_expected_signatures() { pss_profile(field(test, "sha")), &message, &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_signature, &mut scratch, ) @@ -312,7 +351,6 @@ fn nist_cavp_sha2_siggen_private_operations_match_expected_signatures() { ) .expect("CAVP PSS generated signature must verify"); } - other => panic!("unsupported CAVP SigGen scheme `{other}`"), } } @@ -386,8 +424,7 @@ fn nist_cavp_same_width_private_scratch_rebinds_between_keys() { pss_profile(field(test_b, "sha")), &message, &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, &mut scratch, ) @@ -411,8 +448,8 @@ fn nist_cavp_sha2_siggen_profile_signing_matches_expected_results() { let mut scratch_signature = vec![0u8; key.public_key().modulus().len()]; let mut scratch = key.private_scratch(); - match field(test, "scheme") { - "pkcs1v15" => { + match siggen_scheme(field(test, "scheme")) { + SigGenScheme::Pkcs1v15 => { pkcs1v15 = pkcs1v15.strict_add(1); let profile = RsaSignatureProfile::pkcs1v15(pkcs1_profile(field(test, "sha"))); key @@ -430,7 +467,7 @@ fn nist_cavp_sha2_siggen_profile_signing_matches_expected_results() { "CAVP PKCS1v1.5 scratch profile signature mismatch" ); } - "pss" => { + SigGenScheme::Pss => { pss = pss.strict_add(1); let salt_len = hex_to_vec(field(test, "salt")).len(); let profile = RsaSignatureProfile::pss_with_salt_len(pss_profile(field(test, "sha")), salt_len); @@ -449,7 +486,6 @@ fn nist_cavp_sha2_siggen_profile_signing_matches_expected_results() { .verify_signature(profile, &message, &scratch_signature) .expect("CAVP PSS scratch profile signature must verify"); } - other => panic!("unsupported CAVP SigGen scheme `{other}`"), } } @@ -515,12 +551,12 @@ fn nist_cavp_supported_sha2_sigver_subset_matches_expected_results() { ) .is_ok(); - match field(test, "result") { - "P" => { + match cavp_result(field(test, "result")) { + CavpResult::Pass => { counts.valid = counts.valid.strict_add(1); assert!(verified, "CAVP tcId {} rejected valid signature", test["tc_id"]); } - "F" => { + CavpResult::Fail => { counts.invalid = counts.invalid.strict_add(1); assert!( !verified, @@ -528,7 +564,6 @@ fn nist_cavp_supported_sha2_sigver_subset_matches_expected_results() { test["tc_id"], scheme ); } - other => panic!("unknown CAVP result `{other}`"), } } diff --git a/tests/rsa_profile_confusion.rs b/tests/rsa_profile_confusion.rs index 03bcd157..dc94afb5 100644 --- a/tests/rsa_profile_confusion.rs +++ b/tests/rsa_profile_confusion.rs @@ -17,16 +17,16 @@ fn assert_rejects(result: Result<(), VerificationError>) { #[test] fn rsa_signatures_reject_padding_profile_confusion() { - let key = RsaPublicKey::from_spki_der(RSA3072_SPKI).unwrap(); + let key = RsaPublicKey::from_spki_der(RSA3072_SPKI).expect("RSA-3072 SPKI fixture must parse"); let pss = RsaSignatureProfile::pss(RsaPssProfile::Sha256); let pkcs1v15 = RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256); - assert!(key.verify_signature(pss, PSS_MESSAGE, RSA3072_PSS_SHA256).is_ok()); - assert!( - key - .verify_signature(pkcs1v15, PKCS1V15_MESSAGE, RSA3072_PKCS1V15_SHA256) - .is_ok() - ); + key + .verify_signature(pss, PSS_MESSAGE, RSA3072_PSS_SHA256) + .expect("RSA-PSS fixture signature must verify with its matching profile"); + key + .verify_signature(pkcs1v15, PKCS1V15_MESSAGE, RSA3072_PKCS1V15_SHA256) + .expect("PKCS#1 v1.5 fixture signature must verify with its matching profile"); assert_rejects(key.verify_signature(pkcs1v15, PSS_MESSAGE, RSA3072_PSS_SHA256)); assert_rejects(key.verify_signature(pss, PKCS1V15_MESSAGE, RSA3072_PKCS1V15_SHA256)); @@ -36,19 +36,15 @@ fn rsa_signatures_reject_padding_profile_confusion() { #[test] fn rsa_protocol_scheme_helpers_reject_signature_profile_confusion() { - let key = RsaX509PublicKey::from_spki_der(RSA3072_SPKI).unwrap(); - let raw_key = RsaPublicKey::from_spki_der(RSA3072_SPKI).unwrap(); - - assert!( - key - .verify_tls13_signature_scheme(0x0804, PSS_MESSAGE, RSA3072_PSS_SHA256) - .is_ok() - ); - assert!( - key - .verify_tls_certificate_signature_scheme(0x0401, PKCS1V15_MESSAGE, RSA3072_PKCS1V15_SHA256) - .is_ok() - ); + let key = RsaX509PublicKey::from_spki_der(RSA3072_SPKI).expect("RSA-3072 X.509 SPKI fixture must parse"); + let raw_key = RsaPublicKey::from_spki_der(RSA3072_SPKI).expect("RSA-3072 SPKI fixture must parse"); + + key + .verify_tls13_signature_scheme(0x0804, PSS_MESSAGE, RSA3072_PSS_SHA256) + .expect("TLS 1.3 rsa_pss_rsae_sha256 fixture must verify"); + key + .verify_tls_certificate_signature_scheme(0x0401, PKCS1V15_MESSAGE, RSA3072_PKCS1V15_SHA256) + .expect("TLS rsa_pkcs1_sha256 certificate fixture must verify"); assert_rejects(key.verify_tls13_signature_scheme(0x0804, PKCS1V15_MESSAGE, RSA3072_PKCS1V15_SHA256)); assert_rejects(key.verify_tls_certificate_signature_scheme(0x0401, PSS_MESSAGE, RSA3072_PSS_SHA256)); diff --git a/tests/rsa_public_key.rs b/tests/rsa_public_key.rs index 87d72486..86e95e1c 100644 --- a/tests/rsa_public_key.rs +++ b/tests/rsa_public_key.rs @@ -9,7 +9,7 @@ use core::{ use std::{ fs, path::PathBuf, - process::{self, Command}, + process::{self, Command, Output}, sync::OnceLock, time::{SystemTime, UNIX_EPOCH}, }; @@ -71,22 +71,27 @@ const RSA8192_PKCS1V15_SHA256: &[u8] = include_bytes!("../benches/rsa_fixtures/r fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("a short DER length must fit in one byte")]; } let bytes = len.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .expect("a DER length of at least 128 must contain a nonzero byte"); let len_bytes = &bytes[first_nonzero..]; - let mut out = Vec::with_capacity(1 + len_bytes.len()); - out.push(0x80 | len_bytes.len() as u8); + let mut out = Vec::with_capacity(1usize.strict_add(len_bytes.len())); + out.push(0x80 | u8::try_from(len_bytes.len()).expect("a usize DER length prefix must fit in one byte")); out.extend_from_slice(len_bytes); out } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + der_len(value.len()).len() + value.len()); + let encoded_len = der_len(value.len()); + let capacity = 1usize.strict_add(encoded_len.len()).strict_add(value.len()); + let mut out = Vec::with_capacity(capacity); out.push(tag); - out.extend_from_slice(&der_len(value.len())); + out.extend_from_slice(&encoded_len); out.extend_from_slice(value); out } @@ -99,7 +104,7 @@ fn tlv_with_leading_zero_long_len(der: &[u8]) -> Vec { let mut out = Vec::with_capacity(der.len().strict_add(1)); out.push(tag); - out.push(0x80 | (len_len.strict_add(1) as u8)); + out.push(0x80 | u8::try_from(len_len.strict_add(1)).expect("a DER length-of-length must fit in one byte")); out.push(0); out.extend_from_slice(&der[2..]); out @@ -170,7 +175,7 @@ fn context_constructed(index: u8, value: &[u8]) -> Vec { fn integer_unsigned(value: &[u8]) -> Vec { let first_nonzero = value.iter().position(|&byte| byte != 0); let value = first_nonzero.map_or(&[0u8][..], |index| &value[index..]); - let mut encoded = Vec::with_capacity(value.len() + usize::from(value[0] & 0x80 != 0)); + let mut encoded = Vec::with_capacity(value.len().strict_add(usize::from(value[0] & 0x80 != 0))); if value[0] & 0x80 != 0 { encoded.push(0); } @@ -188,11 +193,12 @@ fn algorithm_identifier(algorithm_oid: &[u8], params: Option<&[u8]>) -> Vec } fn hex_to_vec(hex: &str) -> Vec { - assert_eq!(hex.len() % 2, 0); - let mut out = Vec::with_capacity(hex.len() / 2); - for chunk in hex.as_bytes().chunks_exact(2) { - let hi = hex_value(chunk[0]); - let lo = hex_value(chunk[1]); + let (chunks, remainder) = hex.as_bytes().as_chunks::<2>(); + assert!(remainder.is_empty(), "hex input must contain complete byte pairs"); + let mut out = Vec::with_capacity(chunks.len()); + for &[hi, lo] in chunks { + let hi = hex_value(hi).expect("hex fixtures must contain only ASCII hexadecimal digits"); + let lo = hex_value(lo).expect("hex fixtures must contain only ASCII hexadecimal digits"); out.push((hi << 4) | lo); } out @@ -211,12 +217,12 @@ fn cavp_hex_to_vec(hex: &str) -> Vec { hex_to_vec(hex) } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } @@ -246,20 +252,26 @@ fn valid_pkcs1_with_modulus_and_exponent(modulus: &[u8], exponent: &[u8]) -> Vec } fn legacy_public_key_from_pkcs1(der: &[u8]) -> RsaPublicKey { - RsaPublicKey::from_pkcs1_der_with_policy(der, &RsaPublicKeyPolicy::legacy_verification()).unwrap() + RsaPublicKey::from_pkcs1_der_with_policy(der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("the PKCS#1 fixture must satisfy the legacy verification policy") } fn legacy_public_key_from_spki(der: &[u8]) -> RsaPublicKey { - RsaPublicKey::from_spki_der_with_policy(der, &RsaPublicKeyPolicy::legacy_verification()).unwrap() + RsaPublicKey::from_spki_der_with_policy(der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("the SPKI fixture must satisfy the legacy verification policy") } fn legacy_x509_public_key_from_spki(der: &[u8]) -> RsaX509PublicKey { - RsaX509PublicKey::from_spki_der_with_policy(der, &RsaPublicKeyPolicy::legacy_verification()).unwrap() + RsaX509PublicKey::from_spki_der_with_policy(der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("the X.509 SPKI fixture must satisfy the legacy verification policy") } fn exponent_bytes(exponent: u64) -> Vec { let bytes = exponent.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap_or(bytes.len() - 1); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .unwrap_or_else(|| bytes.len().strict_sub(1)); bytes[first_nonzero..].to_vec() } @@ -283,12 +295,16 @@ fn x509_pss_algorithm(profile: RsaPssProfile, salt_len: usize, trailer: Option Vec params.extend_from_slice(&context_constructed(1, &mgf1_sha256_without_params)); params.extend_from_slice(&context_constructed( 2, - &integer_unsigned(&exponent_bytes(u64::try_from(salt_len).unwrap())), + &integer_unsigned(&exponent_bytes( + u64::try_from(salt_len).expect("an X.509 PSS salt length must fit in u64"), + )), )); algorithm_identifier(ID_RSASSA_PSS_OID, Some(&sequence(¶ms))) } fn x509_certificate(tbs_certificate_der: &[u8], signature_algorithm_der: &[u8], signature: &[u8]) -> Vec { - let mut signature_value = Vec::with_capacity(signature.len() + 1); + let mut signature_value = Vec::with_capacity(signature.len().strict_add(1)); signature_value.push(0); signature_value.extend_from_slice(signature); @@ -386,27 +404,27 @@ fn rustcrypto_fixture_private_key() -> RustCryptoRsaPrivateKey { b"00d397b84d98a4c26138ed1b695a8106ead91d553bf06041b62d3fdc50a041e222b8f4529689c1b82c5e71554f5dd69fa2f4b6158cf0dbeb57811a0fc327e1f28e74fe74d3bc166c1eabdc1b8b57b934ca8be5b00b4f29975bcc99acaf415b59bb28a6782bb41a2c3c2976b3c18dbadef62f00c6bb226640095096c0cc60d22fe7ef987d75c6a81b10d96bf292028af110dc7cc1bbc43d22adab379a0cd5d8078cc780ff5cd6209dea34c922cf784f7717e428d75b5aec8ff30e5f0141510766e2e0ab8d473c84e8710b2b98227c3db095337ad3452f19e2b9bfbccdd8148abf6776fa552775e6e75956e45229ae5a9c46949bab1e622f0e48f56524a84ed3483b", 16, ) - .unwrap(), - BigUint::parse_bytes(b"010001", 16).unwrap(), + .expect("the fixed RSA modulus must be valid hexadecimal"), + BigUint::parse_bytes(b"010001", 16).expect("the fixed RSA exponent must be valid hexadecimal"), BigUint::parse_bytes( b"00c4e70c689162c94c660828191b52b4d8392115df486a9adbe831e458d73958320dc1b755456e93701e9702d76fb0b92f90e01d1fe248153281fe79aa9763a92fae69d8d7ecd144de29fa135bd14f9573e349e45031e3b76982f583003826c552e89a397c1a06bd2163488630d92e8c2bb643d7abef700da95d685c941489a46f54b5316f62b5d2c3a7f1bbd134cb37353a44683fdc9d95d36458de22f6c44057fe74a0a436c4308f73f4da42f35c47ac16a7138d483afc91e41dc3a1127382e0c0f5119b0221b4fc639d6b9c38177a6de9b526ebd88c38d7982c07f98a0efd877d508aae275b946915c02e2e1106d175d74ec6777f5e80d12c053d9c7be1e341", 16, ) - .unwrap(), + .expect("the fixed RSA private exponent must be valid hexadecimal"), vec![ BigUint::parse_bytes( b"00f827bbf3a41877c7cc59aebf42ed4b29c32defcb8ed96863d5b090a05a8930dd624a21c9dcf9838568fdfa0df65b8462a5f2ac913d6c56f975532bd8e78fb07bd405ca99a484bcf59f019bbddcb3933f2bce706300b4f7b110120c5df9018159067c35da3061a56c8635a52b54273b31271b4311f0795df6021e6355e1a42e61", 16, ) - .unwrap(), + .expect("the fixed first RSA prime must be valid hexadecimal"), BigUint::parse_bytes( b"00da4817ce0089dd36f2ade6a3ff410c73ec34bf1b4f6bda38431bfede11cef1f7f6efa70e5f8063a3b1f6e17296ffb15feefa0912a0325b8d1fd65a559e717b5b961ec345072e0ec5203d03441d29af4d64054a04507410cf1da78e7b6119d909ec66e6ad625bf995b279a4b3c5be7d895cd7c5b9c4c497fde730916fcdb4e41b", 16, ) - .unwrap(), + .expect("the fixed second RSA prime must be valid hexadecimal"), ], ) - .unwrap() + .expect("the fixed RSA components must form a valid private key") } fn x509_sha256_rsa_self_signed_certificate(params: Option<&[u8]>) -> (Vec, Vec) { @@ -552,7 +570,7 @@ eb10e16719b6b54d1768dc5278e6bcebc67d45226ab0164ede685b74a3d53eb14bcdfb", } fn spki_for_pkcs1_with_algorithm(pkcs1: &[u8], algorithm: &[u8]) -> Vec { - let mut subject_public_key = Vec::with_capacity(pkcs1.len() + 1); + let mut subject_public_key = Vec::with_capacity(pkcs1.len().strict_add(1)); subject_public_key.push(0); subject_public_key.extend_from_slice(pkcs1); @@ -600,58 +618,78 @@ fn assert_aws_lc_rs_pkcs1v15_sha256(spki: &[u8], message: &[u8], signature: &[u8 fn assert_ring_cavp(scheme: &str, sha: &str, pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { let actual = match (scheme, sha) { - ("pss", "SHA256") => ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PSS_2048_8192_SHA256, pkcs1) - .verify(message, signature) - .is_ok(), - ("pss", "SHA384") => ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PSS_2048_8192_SHA384, pkcs1) - .verify(message, signature) - .is_ok(), - ("pss", "SHA512") => ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PSS_2048_8192_SHA512, pkcs1) - .verify(message, signature) - .is_ok(), - ("pkcs1v15", "SHA256") => { + ("pss", "SHA256") => Some( + ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PSS_2048_8192_SHA256, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pss", "SHA384") => Some( + ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PSS_2048_8192_SHA384, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pss", "SHA512") => Some( + ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PSS_2048_8192_SHA512, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pkcs1v15", "SHA256") => Some( ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PKCS1_2048_8192_SHA256, pkcs1) .verify(message, signature) - .is_ok() - } - ("pkcs1v15", "SHA384") => { + .is_ok(), + ), + ("pkcs1v15", "SHA384") => Some( ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PKCS1_2048_8192_SHA384, pkcs1) .verify(message, signature) - .is_ok() - } - ("pkcs1v15", "SHA512") => { + .is_ok(), + ), + ("pkcs1v15", "SHA512") => Some( ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PKCS1_2048_8192_SHA512, pkcs1) .verify(message, signature) - .is_ok() - } - other => panic!("unsupported ring CAVP profile {other:?}"), - }; + .is_ok(), + ), + _ => None, + } + .expect("CAVP fixtures must use a ring-supported RSA/SHA-2 profile"); assert_eq!(actual, expected, "ring mismatch for {scheme}/{sha}"); } #[cfg(not(any(target_arch = "s390x", target_arch = "powerpc64")))] fn assert_aws_lc_rs_cavp(scheme: &str, sha: &str, pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { let actual = match (scheme, sha) { - ("pss", "SHA256") => aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA256, pkcs1) - .verify(message, signature) - .is_ok(), - ("pss", "SHA384") => aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA384, pkcs1) - .verify(message, signature) - .is_ok(), - ("pss", "SHA512") => aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA512, pkcs1) - .verify(message, signature) - .is_ok(), - ("pkcs1v15", "SHA256") => aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA256, pkcs1) - .verify(message, signature) - .is_ok(), - ("pkcs1v15", "SHA384") => aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA384, pkcs1) - .verify(message, signature) - .is_ok(), - ("pkcs1v15", "SHA512") => aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA512, pkcs1) - .verify(message, signature) - .is_ok(), - other => panic!("unsupported aws-lc-rs CAVP profile {other:?}"), - }; + ("pss", "SHA256") => Some( + aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA256, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pss", "SHA384") => Some( + aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA384, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pss", "SHA512") => Some( + aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA512, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pkcs1v15", "SHA256") => Some( + aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA256, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pkcs1v15", "SHA384") => Some( + aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA384, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pkcs1v15", "SHA512") => Some( + aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA512, pkcs1) + .verify(message, signature) + .is_ok(), + ), + _ => None, + } + .expect("CAVP fixtures must use an aws-lc-rs-supported RSA/SHA-2 profile"); assert_eq!(actual, expected, "aws-lc-rs mismatch for {scheme}/{sha}"); } @@ -689,30 +727,30 @@ impl Drop for AwsLcMdCtx { } #[cfg(not(any(target_arch = "s390x", target_arch = "powerpc64")))] -fn aws_lc_md(sha: &str) -> *const aws_lc::EVP_MD { +fn aws_lc_md(sha: &str) -> Option<*const aws_lc::EVP_MD> { match sha { "SHA256" => { // SAFETY: Obtains AWS-LC's static SHA-256 EVP_MD descriptor because: // 1. The function takes no caller-owned pointers. // 2. AWS-LC returns a process-static descriptor that must not be freed. // 3. The descriptor is only passed back into AWS-LC verification APIs. - unsafe { aws_lc::EVP_sha256() } + Some(unsafe { aws_lc::EVP_sha256() }) } "SHA384" => { // SAFETY: Obtains AWS-LC's static SHA-384 EVP_MD descriptor because: // 1. The function takes no caller-owned pointers. // 2. AWS-LC returns a process-static descriptor that must not be freed. // 3. The descriptor is only passed back into AWS-LC verification APIs. - unsafe { aws_lc::EVP_sha384() } + Some(unsafe { aws_lc::EVP_sha384() }) } "SHA512" => { // SAFETY: Obtains AWS-LC's static SHA-512 EVP_MD descriptor because: // 1. The function takes no caller-owned pointers. // 2. AWS-LC returns a process-static descriptor that must not be freed. // 3. The descriptor is only passed back into AWS-LC verification APIs. - unsafe { aws_lc::EVP_sha512() } + Some(unsafe { aws_lc::EVP_sha512() }) } - other => panic!("unsupported AWS-LC sys hash `{other}`"), + _ => None, } } @@ -751,7 +789,7 @@ fn aws_lc_sys_verify( ) -> Option { let key = aws_lc_parse_public_key(spki)?; let ctx = aws_lc_new_md_ctx()?; - let md = aws_lc_md(sha); + let md = aws_lc_md(sha)?; let mut pctx = ptr::null_mut(); // SAFETY: Initializes an AWS-LC digest verification context because: @@ -759,7 +797,8 @@ fn aws_lc_sys_verify( // 2. `md` is a live process-static descriptor returned by AWS-LC for the selected SHA-2 digest. // 3. `pctx` points to stack storage where AWS-LC writes a context borrowed by `ctx`; it is not // freed separately. - let init_rc = unsafe { aws_lc::EVP_DigestVerifyInit(ctx.0.as_ptr(), &mut pctx, md, ptr::null_mut(), key.0.as_ptr()) }; + let init_rc = + unsafe { aws_lc::EVP_DigestVerifyInit(ctx.0.as_ptr(), &raw mut pctx, md, ptr::null_mut(), key.0.as_ptr()) }; if init_rc != 1 || pctx.is_null() { return None; } @@ -767,7 +806,7 @@ fn aws_lc_sys_verify( let padding = match scheme { "pss" => aws_lc::RSA_PKCS1_PSS_PADDING, "pkcs1v15" => aws_lc::RSA_PKCS1_PADDING, - other => panic!("unsupported AWS-LC sys RSA scheme `{other}`"), + _ => return None, }; // SAFETY: Configures RSA verification padding on AWS-LC's borrowed PKEY context because: @@ -824,7 +863,7 @@ fn assert_aws_lc_sys_cavp( ) { let spki = spki_for_pkcs1(pkcs1); let actual = aws_lc_sys_verify(scheme, sha, salt_len, &spki, message, signature) - .unwrap_or_else(|| panic!("AWS-LC sys setup failed for {scheme}/{sha} salt_len={salt_len:?}")); + .expect("CAVP fixtures must select supported AWS-LC RSA/SHA-2 parameters"); assert_eq!(actual, expected, "AWS-LC sys mismatch for {scheme}/{sha}"); } @@ -850,47 +889,58 @@ fn assert_rustcrypto_cavp( signature: &[u8], expected: bool, ) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the CAVP PKCS#1 public key must parse"); let actual = match (scheme, sha) { ("pss", "SHA256") => { - let key = RustCryptoPssVerifyingKey::::new_with_salt_len(key, salt_len.unwrap()); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); - key.verify(message, &signature).is_ok() + let key = RustCryptoPssVerifyingKey::::new_with_salt_len( + key, + salt_len.expect("a CAVP PSS case must provide a salt length"), + ); + let signature = RustCryptoPssSignature::try_from(signature).expect("the CAVP PSS signature width must match"); + Some(key.verify(message, &signature).is_ok()) } ("pss", "SHA384") => { - let key = RustCryptoPssVerifyingKey::::new_with_salt_len(key, salt_len.unwrap()); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); - key.verify(message, &signature).is_ok() + let key = RustCryptoPssVerifyingKey::::new_with_salt_len( + key, + salt_len.expect("a CAVP PSS case must provide a salt length"), + ); + let signature = RustCryptoPssSignature::try_from(signature).expect("the CAVP PSS signature width must match"); + Some(key.verify(message, &signature).is_ok()) } ("pss", "SHA512") => { - let key = RustCryptoPssVerifyingKey::::new_with_salt_len(key, salt_len.unwrap()); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); - key.verify(message, &signature).is_ok() + let key = RustCryptoPssVerifyingKey::::new_with_salt_len( + key, + salt_len.expect("a CAVP PSS case must provide a salt length"), + ); + let signature = RustCryptoPssSignature::try_from(signature).expect("the CAVP PSS signature width must match"); + Some(key.verify(message, &signature).is_ok()) } ("pkcs1v15", "SHA256") => { let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); - key.verify(message, &signature).is_ok() + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the CAVP PKCS#1 v1.5 signature width must match"); + Some(key.verify(message, &signature).is_ok()) } ("pkcs1v15", "SHA384") => { let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); - key.verify(message, &signature).is_ok() + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the CAVP PKCS#1 v1.5 signature width must match"); + Some(key.verify(message, &signature).is_ok()) } ("pkcs1v15", "SHA512") => { let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); - key.verify(message, &signature).is_ok() + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the CAVP PKCS#1 v1.5 signature width must match"); + Some(key.verify(message, &signature).is_ok()) } - other => panic!("unsupported RustCrypto CAVP profile {other:?}"), - }; + _ => None, + } + .expect("CAVP fixtures must use a RustCrypto-supported RSA/SHA-2 profile"); assert_eq!(actual, expected, "RustCrypto mismatch for {scheme}/{sha}"); } fn cavp_field<'a>(value: &'a serde_json::Value, name: &'static str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing CAVP string field `{name}`")) + value[name].as_str().expect("a required CAVP field must be a string") } fn cavp_public_exponent(bytes: &[u8]) -> u64 { @@ -913,81 +963,92 @@ fn cavp_rscrypto_result( signature: &[u8], ) -> Result<(), rscrypto::VerificationError> { let profile = match (scheme, sha) { - ("pss", "SHA256") => RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, salt_len.unwrap()), - ("pss", "SHA384") => RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha384, salt_len.unwrap()), - ("pss", "SHA512") => RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha512, salt_len.unwrap()), - ("pkcs1v15", "SHA256") => RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), - ("pkcs1v15", "SHA384") => RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha384), - ("pkcs1v15", "SHA512") => RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha512), - other => panic!("unsupported rscrypto CAVP profile {other:?}"), - }; + ("pss", "SHA256") => { + salt_len.map(|salt_len| RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, salt_len)) + } + ("pss", "SHA384") => { + salt_len.map(|salt_len| RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha384, salt_len)) + } + ("pss", "SHA512") => { + salt_len.map(|salt_len| RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha512, salt_len)) + } + ("pkcs1v15", "SHA256") => Some(RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256)), + ("pkcs1v15", "SHA384") => Some(RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha384)), + ("pkcs1v15", "SHA512") => Some(RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha512)), + _ => None, + } + .expect("CAVP fixtures must use a supported RSA/SHA-2 profile and provide a PSS salt length"); key.verify_signature(profile, message, signature) } fn assert_rustcrypto_pss_sha256(pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the PKCS#1 public-key fixture must parse"); let key = RustCryptoPssVerifyingKey::::new(key); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); + let signature = RustCryptoPssSignature::try_from(signature).expect("the PSS signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } #[cfg(feature = "getrandom")] fn assert_rustcrypto_pss_sha384(pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the PKCS#1 public-key fixture must parse"); let key = RustCryptoPssVerifyingKey::::new(key); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); + let signature = RustCryptoPssSignature::try_from(signature).expect("the PSS signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } #[cfg(feature = "getrandom")] fn assert_rustcrypto_pss_sha512(pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the PKCS#1 public-key fixture must parse"); let key = RustCryptoPssVerifyingKey::::new(key); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); + let signature = RustCryptoPssSignature::try_from(signature).expect("the PSS signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } fn assert_rustcrypto_pkcs1v15_sha256(pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the PKCS#1 public-key fixture must parse"); let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the PKCS#1 v1.5 signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } #[cfg(feature = "getrandom")] fn assert_rustcrypto_pkcs1v15_sha384(pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the PKCS#1 public-key fixture must parse"); let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the PKCS#1 v1.5 signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } #[cfg(feature = "getrandom")] fn assert_rustcrypto_pkcs1v15_sha512(pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the PKCS#1 public-key fixture must parse"); let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the PKCS#1 v1.5 signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } fn assert_rustcrypto_spki_pss_sha256(spki: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_public_key_der(spki).unwrap(); + let key = RustCryptoRsaPublicKey::from_public_key_der(spki).expect("the SPKI public-key fixture must parse"); let key = RustCryptoPssVerifyingKey::::new(key); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); + let signature = RustCryptoPssSignature::try_from(signature).expect("the PSS signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } fn assert_rustcrypto_spki_pkcs1v15_sha256(spki: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_public_key_der(spki).unwrap(); + let key = RustCryptoRsaPublicKey::from_public_key_der(spki).expect("the SPKI public-key fixture must parse"); let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the PKCS#1 v1.5 signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } @@ -1012,7 +1073,16 @@ fn assert_openssl_signature( assert_eq!(actual, expected); } -#[allow(clippy::std_instead_of_core)] +fn openssl_output(command: &mut Command) -> Option { + match command.output() { + Ok(output) => Some(output), + Err(error) => { + eprintln!("OpenSSL oracle unavailable because the process could not start: {error}"); + None + } + } +} + fn openssl_verify( digest_arg: &'static str, spki: &[u8], @@ -1025,9 +1095,9 @@ fn openssl_verify( let msg_path = openssl_temp_path(&id, "msg.bin"); let sig_path = openssl_temp_path(&id, "sig.bin"); - fs::write(&key_path, spki).unwrap(); - fs::write(&msg_path, message).unwrap(); - fs::write(&sig_path, signature).unwrap(); + fs::write(&key_path, spki).expect("the OpenSSL SPKI fixture must be written"); + fs::write(&msg_path, message).expect("the OpenSSL message fixture must be written"); + fs::write(&sig_path, signature).expect("the OpenSSL signature fixture must be written"); let mut command = Command::new("openssl"); command @@ -1039,17 +1109,13 @@ fn openssl_verify( for sigopt in sigopts { command.args(["-sigopt", sigopt]); } - let output = command.arg(&msg_path).output(); + let output = openssl_output(command.arg(&msg_path)); - let _ = fs::remove_file(&key_path); - let _ = fs::remove_file(&msg_path); - let _ = fs::remove_file(&sig_path); + fs::remove_file(&key_path).expect("the temporary OpenSSL SPKI fixture must be removed"); + fs::remove_file(&msg_path).expect("the temporary OpenSSL message fixture must be removed"); + fs::remove_file(&sig_path).expect("the temporary OpenSSL signature fixture must be removed"); - let output = match output { - Ok(output) => output, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, - Err(error) => panic!("failed to run openssl: {error}"), - }; + let output = output?; if output.status.success() { return Some(true); @@ -1057,18 +1123,15 @@ fn openssl_verify( let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); - if stdout.contains("Verification failure") || stderr.contains("bad signature") { - return Some(false); - } - - panic!( + assert!( + stdout.contains("Verification failure") || stderr.contains("bad signature"), "openssl RSA verify failed unexpectedly: status={:?} stdout={stdout:?} stderr={stderr:?}", output.status.code() ); + Some(false) } #[cfg(feature = "getrandom")] -#[allow(clippy::std_instead_of_core)] fn openssl_oaep_crypt( operation: &'static str, key_der: &[u8], @@ -1080,8 +1143,8 @@ fn openssl_oaep_crypt( let key_path = openssl_temp_path(&id, "oaep-key.der"); let input_path = openssl_temp_path(&id, "oaep-input.bin"); - fs::write(&key_path, key_der).unwrap(); - fs::write(&input_path, input).unwrap(); + fs::write(&key_path, key_der).expect("the OpenSSL OAEP key fixture must be written"); + fs::write(&input_path, input).expect("the OpenSSL OAEP input fixture must be written"); let mut command = Command::new("openssl"); command @@ -1090,34 +1153,29 @@ fn openssl_oaep_crypt( if public_key { command.arg("-pubin"); } - let output = command - .arg("-in") - .arg(&input_path) - .args(["-pkeyopt", "rsa_padding_mode:oaep", "-pkeyopt"]) - .arg(format!("rsa_mgf1_md:{digest}")) - .arg("-pkeyopt") - .arg(format!("rsa_oaep_md:{digest}")) - .output(); - - let _ = fs::remove_file(&key_path); - let _ = fs::remove_file(&input_path); - - let output = match output { - Ok(output) => output, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, - Err(error) => panic!("failed to run openssl: {error}"), - }; + let output = openssl_output( + command + .arg("-in") + .arg(&input_path) + .args(["-pkeyopt", "rsa_padding_mode:oaep", "-pkeyopt"]) + .arg(format!("rsa_mgf1_md:{digest}")) + .arg("-pkeyopt") + .arg(format!("rsa_oaep_md:{digest}")), + ); - if output.status.success() { - return Some(output.stdout); - } + fs::remove_file(&key_path).expect("the temporary OpenSSL OAEP key fixture must be removed"); + fs::remove_file(&input_path).expect("the temporary OpenSSL OAEP input fixture must be removed"); + + let output = output?; let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); - panic!( + assert!( + output.status.success(), "openssl RSA OAEP {operation} failed unexpectedly: status={:?} stdout={stdout:?} stderr={stderr:?}", output.status.code() ); + Some(output.stdout) } #[cfg(feature = "getrandom")] @@ -1131,14 +1189,13 @@ fn openssl_oaep_decrypt(pkcs1_private_key: &[u8], ciphertext: &[u8], digest: &'s } #[cfg(feature = "getrandom")] -#[allow(clippy::std_instead_of_core)] fn openssl_pkcs1v15_crypt(operation: &'static str, key_der: &[u8], input: &[u8], public_key: bool) -> Option> { let id = openssl_temp_id(); let key_path = openssl_temp_path(&id, "pkcs1v15-key.der"); let input_path = openssl_temp_path(&id, "pkcs1v15-input.bin"); - fs::write(&key_path, key_der).unwrap(); - fs::write(&input_path, input).unwrap(); + fs::write(&key_path, key_der).expect("the OpenSSL PKCS#1 v1.5 key fixture must be written"); + fs::write(&input_path, input).expect("the OpenSSL PKCS#1 v1.5 input fixture must be written"); let mut command = Command::new("openssl"); command @@ -1147,31 +1204,26 @@ fn openssl_pkcs1v15_crypt(operation: &'static str, key_der: &[u8], input: &[u8], if public_key { command.arg("-pubin"); } - let output = command - .arg("-in") - .arg(&input_path) - .args(["-pkeyopt", "rsa_padding_mode:pkcs1"]) - .output(); - - let _ = fs::remove_file(&key_path); - let _ = fs::remove_file(&input_path); - - let output = match output { - Ok(output) => output, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, - Err(error) => panic!("failed to run openssl: {error}"), - }; + let output = openssl_output( + command + .arg("-in") + .arg(&input_path) + .args(["-pkeyopt", "rsa_padding_mode:pkcs1"]), + ); - if output.status.success() { - return Some(output.stdout); - } + fs::remove_file(&key_path).expect("the temporary OpenSSL PKCS#1 v1.5 key fixture must be removed"); + fs::remove_file(&input_path).expect("the temporary OpenSSL PKCS#1 v1.5 input fixture must be removed"); + + let output = output?; let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); - panic!( + assert!( + output.status.success(), "openssl RSA PKCS#1 v1.5 {operation} failed unexpectedly: status={:?} stdout={stdout:?} stderr={stderr:?}", output.status.code() ); + Some(output.stdout) } #[cfg(feature = "getrandom")] @@ -1199,10 +1251,12 @@ fn assert_generated_pkcs1v15_encryption_external_oracles( key .public_key() .encrypt_pkcs1v15_with_random_fill(plaintext, &mut ciphertext, fill_rsa_random_with(0x37)) - .unwrap(); + .expect("deterministic PKCS#1 v1.5 encryption must succeed"); let rustcrypto_decrypting_key = RustCryptoPkcs1v15DecryptingKey::new(rustcrypto_private_key.clone()); - let rustcrypto_decrypted = rustcrypto_decrypting_key.decrypt(&ciphertext).unwrap(); + let rustcrypto_decrypted = rustcrypto_decrypting_key + .decrypt(&ciphertext) + .expect("RustCrypto must decrypt the matching deterministic PKCS#1 v1.5 ciphertext"); assert_eq!(rustcrypto_decrypted, plaintext); if let Some(openssl_decrypted) = openssl_pkcs1v15_decrypt(pkcs1, &ciphertext) { @@ -1217,7 +1271,7 @@ fn assert_generated_pkcs1v15_encryption_external_oracles( let mut rscrypto_decrypted = vec![0u8; key.signature_len()]; let rscrypto_decrypted_len = key .decrypt_pkcs1v15(&openssl_ciphertext, &mut rscrypto_decrypted) - .unwrap(); + .expect("rscrypto must decrypt the matching OpenSSL PKCS#1 v1.5 ciphertext"); assert_eq!(&rscrypto_decrypted[..rscrypto_decrypted_len], plaintext); } else { eprintln!( @@ -1226,69 +1280,69 @@ fn assert_generated_pkcs1v15_encryption_external_oracles( } } -#[allow(clippy::std_instead_of_core)] fn openssl_private_key_to_spki_der(private_key_der: &[u8]) -> Option> { let id = openssl_temp_id(); let key_path = openssl_temp_path(&id, "private-key.der"); - fs::write(&key_path, private_key_der).unwrap(); + fs::write(&key_path, private_key_der).expect("the OpenSSL private-key fixture must be written"); - let output = Command::new("openssl") - .args(["pkey", "-inform", "DER", "-in"]) - .arg(&key_path) - .args(["-pubout", "-outform", "DER"]) - .output(); + let output = openssl_output( + Command::new("openssl") + .args(["pkey", "-inform", "DER", "-in"]) + .arg(&key_path) + .args(["-pubout", "-outform", "DER"]), + ); - let _ = fs::remove_file(&key_path); + fs::remove_file(&key_path).expect("the temporary OpenSSL private-key fixture must be removed"); - let output = match output { - Ok(output) => output, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, - Err(error) => panic!("failed to run openssl: {error}"), - }; - - if output.status.success() { - return Some(output.stdout); - } + let output = output?; let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); - panic!( + assert!( + output.status.success(), "openssl RSA private-key DER parse failed unexpectedly: status={:?} stdout={stdout:?} stderr={stderr:?}", output.status.code() ); + Some(output.stdout) } fn digest_salt_len(sha: &str) -> usize { match sha { - "SHA256" => 32, - "SHA384" => 48, - "SHA512" => 64, - other => panic!("unsupported SHA-2 profile `{other}`"), + "SHA256" => Some(32), + "SHA384" => Some(48), + "SHA512" => Some(64), + _ => None, } + .expect("CAVP fixtures must use a supported SHA-2 profile") } fn openssl_pss_sigopts(sha: &str, salt_len: usize) -> [&'static str; 3] { let salt_len = match salt_len { - 0 => "rsa_pss_saltlen:0", - 1 => "rsa_pss_saltlen:1", - 24 => "rsa_pss_saltlen:24", - 32 => "rsa_pss_saltlen:32", - 48 => "rsa_pss_saltlen:48", - 64 => "rsa_pss_saltlen:64", - other => panic!("unsupported CAVP PSS salt length `{other}`"), - }; + 0 => Some("rsa_pss_saltlen:0"), + 1 => Some("rsa_pss_saltlen:1"), + 24 => Some("rsa_pss_saltlen:24"), + 32 => Some("rsa_pss_saltlen:32"), + 48 => Some("rsa_pss_saltlen:48"), + 64 => Some("rsa_pss_saltlen:64"), + _ => None, + } + .expect("CAVP fixtures must use a supported PSS salt length"); let mgf1 = match sha { - "SHA256" => "rsa_mgf1_md:sha256", - "SHA384" => "rsa_mgf1_md:sha384", - "SHA512" => "rsa_mgf1_md:sha512", - other => panic!("unsupported OpenSSL PSS hash `{other}`"), - }; + "SHA256" => Some("rsa_mgf1_md:sha256"), + "SHA384" => Some("rsa_mgf1_md:sha384"), + "SHA512" => Some("rsa_mgf1_md:sha512"), + _ => None, + } + .expect("CAVP fixtures must use an OpenSSL-supported PSS hash"); ["rsa_padding_mode:pss", salt_len, mgf1] } fn openssl_temp_id() -> String { - let nanos = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("the system clock must be after the Unix epoch") + .as_nanos(); format!("rscrypto-rsa-{}-{nanos}", process::id()) } @@ -1333,7 +1387,10 @@ fn fixed_width_signature_candidate(material: &[u8], len: usize) -> Vec { return out; } for (index, byte) in out.iter_mut().enumerate() { - *byte = material[index % material.len()]; + let source_index = index + .checked_rem(material.len()) + .expect("nonempty signature material must have a valid remainder"); + *byte = material[source_index]; } out } @@ -1343,7 +1400,7 @@ fn assert_opaque_verification_failure(result: Result<(), rscrypto::VerificationE } fn assert_unsupported_protocol_algorithm(result: Result) { - let err = result.unwrap_err(); + let err = result.expect_err("the protocol algorithm must be rejected as unsupported"); assert_eq!(err, RsaProtocolAlgorithmError::UnsupportedAlgorithm); assert!(!err.is_malformed_algorithm_identifier()); assert!(err.is_unsupported_algorithm()); @@ -1351,7 +1408,7 @@ fn assert_unsupported_protocol_algorithm(result: Result(result: Result) { - let err = result.unwrap_err(); + let err = result.expect_err("the malformed protocol algorithm identifier must be rejected"); assert_eq!(err, RsaProtocolAlgorithmError::MalformedAlgorithmIdentifier); assert!(err.is_malformed_algorithm_identifier()); assert!(!err.is_unsupported_algorithm()); @@ -1389,15 +1446,17 @@ fn spki_public_key_accepts_rsa2048_with_legacy_policy() { #[test] fn private_key_der_exports_roundtrip_with_rustcrypto_rsa() { let rustcrypto_key = rustcrypto_fixture_private_key(); - let rustcrypto_pkcs1 = EncodeRsaPrivateKey::to_pkcs1_der(&rustcrypto_key).unwrap(); - let rustcrypto_pkcs8 = EncodePrivateKey::to_pkcs8_der(&rustcrypto_key).unwrap(); + let rustcrypto_pkcs1 = + EncodeRsaPrivateKey::to_pkcs1_der(&rustcrypto_key).expect("the RustCrypto fixture must encode as PKCS#1"); + let rustcrypto_pkcs8 = + EncodePrivateKey::to_pkcs8_der(&rustcrypto_key).expect("the RustCrypto fixture must encode as PKCS#8"); let rscrypto_from_pkcs1 = RsaPrivateKey::from_pkcs1_der_with_policy(rustcrypto_pkcs1.as_bytes(), &RsaPublicKeyPolicy::legacy_verification()) - .unwrap(); + .expect("rscrypto must decode the RustCrypto PKCS#1 private-key fixture"); let rscrypto_from_pkcs8 = RsaPrivateKey::from_pkcs8_der_with_policy(rustcrypto_pkcs8.as_bytes(), &RsaPublicKeyPolicy::legacy_verification()) - .unwrap(); + .expect("rscrypto must decode the RustCrypto PKCS#8 private-key fixture"); assert_eq!(rscrypto_from_pkcs1.public_key(), rscrypto_from_pkcs8.public_key()); let rscrypto_pkcs1 = rscrypto_from_pkcs1.to_pkcs1_der(); @@ -1448,7 +1507,8 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { impl RngCore for DeterministicRng { fn next_u32(&mut self) -> u32 { - self.next_u64() as u32 + let word = self.next_u64().to_le_bytes(); + u32::from_le_bytes([word[0], word[1], word[2], word[3]]) } fn next_u64(&mut self) -> u64 { @@ -1475,11 +1535,12 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { impl CryptoRng for DeterministicRng {} let rustcrypto_key = rustcrypto_fixture_private_key(); - let rustcrypto_pkcs1 = EncodeRsaPrivateKey::to_pkcs1_der(&rustcrypto_key).unwrap(); + let rustcrypto_pkcs1 = + EncodeRsaPrivateKey::to_pkcs1_der(&rustcrypto_key).expect("the RustCrypto fixture must encode as PKCS#1"); let rustcrypto_public_key = rustcrypto_key.to_public_key(); let rscrypto_key = RsaPrivateKey::from_pkcs1_der_with_policy(rustcrypto_pkcs1.as_bytes(), &RsaPublicKeyPolicy::legacy_verification()) - .unwrap(); + .expect("rscrypto must decode the RustCrypto private-key fixture"); let rscrypto_public_pkcs1 = rscrypto_key.public_key().to_pkcs1_der(); let rscrypto_public_spki = rscrypto_key.public_key().to_spki_der(); let message = b"rscrypto private outputs verified by RustCrypto rsa"; @@ -1487,7 +1548,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { let mut pkcs1v15_signature = vec![0u8; rscrypto_key.signature_len()]; rscrypto_key .sign_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("PKCS#1 v1.5 SHA-256 signing must succeed for the valid fixture"); assert_rustcrypto_pkcs1v15_sha256(&rscrypto_public_pkcs1, message, &pkcs1v15_signature, true); assert_openssl_signature( "-sha256", @@ -1499,7 +1560,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { ); rscrypto_key .sign_pkcs1v15(RsaPkcs1v15Profile::Sha384, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("PKCS#1 v1.5 SHA-384 signing must succeed for the valid fixture"); assert_rustcrypto_pkcs1v15_sha384(&rscrypto_public_pkcs1, message, &pkcs1v15_signature, true); assert_openssl_signature( "-sha384", @@ -1511,7 +1572,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { ); rscrypto_key .sign_pkcs1v15(RsaPkcs1v15Profile::Sha512, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("PKCS#1 v1.5 SHA-512 signing must succeed for the valid fixture"); assert_rustcrypto_pkcs1v15_sha512(&rscrypto_public_pkcs1, message, &pkcs1v15_signature, true); assert_openssl_signature( "-sha512", @@ -1525,7 +1586,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { let mut pss_signature = vec![0u8; rscrypto_key.signature_len()]; rscrypto_key .sign_pss(RsaPssProfile::Sha256, message, &mut pss_signature) - .unwrap(); + .expect("PSS SHA-256 signing must succeed for the valid fixture"); assert_rustcrypto_pss_sha256(&rscrypto_public_pkcs1, message, &pss_signature, true); let openssl_pss_sha256 = openssl_pss_sigopts("SHA256", 32); assert_openssl_signature( @@ -1538,7 +1599,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { ); rscrypto_key .sign_pss(RsaPssProfile::Sha384, message, &mut pss_signature) - .unwrap(); + .expect("PSS SHA-384 signing must succeed for the valid fixture"); assert_rustcrypto_pss_sha384(&rscrypto_public_pkcs1, message, &pss_signature, true); let openssl_pss_sha384 = openssl_pss_sigopts("SHA384", 48); assert_openssl_signature( @@ -1551,7 +1612,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { ); rscrypto_key .sign_pss(RsaPssProfile::Sha512, message, &mut pss_signature) - .unwrap(); + .expect("PSS SHA-512 signing must succeed for the valid fixture"); assert_rustcrypto_pss_sha512(&rscrypto_public_pkcs1, message, &pss_signature, true); let openssl_pss_sha512 = openssl_pss_sigopts("SHA512", 64); assert_openssl_signature( @@ -1571,14 +1632,18 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { rscrypto_key .public_key() .encrypt_oaep($profile, label.as_bytes(), plaintext, &mut ciphertext) - .unwrap(); + .expect("OAEP encryption must succeed for the valid RustCrypto round-trip fixture"); let decrypting_key = RustCryptoOaepDecryptingKey::<$digest>::new_with_label(rustcrypto_key.clone(), label); - let decrypted = decrypting_key.decrypt(&ciphertext).unwrap(); + let decrypted = decrypting_key + .decrypt(&ciphertext) + .expect("RustCrypto must decrypt the matching rscrypto OAEP ciphertext"); assert_eq!(decrypted, plaintext); let encrypting_key = RustCryptoOaepEncryptingKey::<$digest>::new_with_label(rustcrypto_public_key.clone(), label); let mut rng = DeterministicRng { state: $seed }; - let rustcrypto_ciphertext = encrypting_key.encrypt_with_rng(&mut rng, plaintext).unwrap(); + let rustcrypto_ciphertext = encrypting_key + .encrypt_with_rng(&mut rng, plaintext) + .expect("RustCrypto OAEP encryption must succeed for the valid fixture"); let mut rscrypto_decrypted = vec![0u8; rscrypto_key.signature_len()]; let rscrypto_decrypted_len = rscrypto_key .decrypt_oaep( @@ -1587,7 +1652,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { &rustcrypto_ciphertext, &mut rscrypto_decrypted, ) - .unwrap(); + .expect("rscrypto must decrypt the matching RustCrypto OAEP ciphertext"); assert_eq!(&rscrypto_decrypted[..rscrypto_decrypted_len], plaintext); }}; } @@ -1614,7 +1679,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { rscrypto_key .public_key() .encrypt_oaep($profile, b"", openssl_plaintext, &mut ciphertext) - .unwrap(); + .expect("OAEP encryption must succeed for the valid OpenSSL round-trip fixture"); if let Some(decrypted) = openssl_oaep_decrypt(rustcrypto_pkcs1.as_bytes(), &ciphertext, $digest) { assert_eq!(decrypted, openssl_plaintext); } else { @@ -1625,7 +1690,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { let mut rscrypto_decrypted = vec![0u8; rscrypto_key.signature_len()]; let rscrypto_decrypted_len = rscrypto_key .decrypt_oaep($profile, b"", &openssl_ciphertext, &mut rscrypto_decrypted) - .unwrap(); + .expect("rscrypto must decrypt the matching OpenSSL OAEP ciphertext"); assert_eq!(&rscrypto_decrypted[..rscrypto_decrypted_len], openssl_plaintext); } else { eprintln!("skipping OpenSSL RSA OAEP encrypt differential check because `openssl` is not available"); @@ -1641,7 +1706,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { rscrypto_key .public_key() .encrypt_pkcs1v15_with_random_fill(pkcs1v15_plaintext, &mut pkcs1v15_ciphertext, fill_rsa_random_with(0x6d)) - .unwrap(); + .expect("PKCS#1 v1.5 encryption must succeed for the valid OpenSSL round-trip fixture"); if let Some(decrypted) = openssl_pkcs1v15_decrypt(rustcrypto_pkcs1.as_bytes(), &pkcs1v15_ciphertext) { assert_eq!(decrypted, pkcs1v15_plaintext); } else { @@ -1652,7 +1717,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { let mut rscrypto_decrypted = vec![0u8; rscrypto_key.signature_len()]; let rscrypto_decrypted_len = rscrypto_key .decrypt_pkcs1v15(&openssl_ciphertext, &mut rscrypto_decrypted) - .unwrap(); + .expect("rscrypto must decrypt the matching OpenSSL PKCS#1 v1.5 ciphertext"); assert_eq!(&rscrypto_decrypted[..rscrypto_decrypted_len], pkcs1v15_plaintext); } else { eprintln!("skipping OpenSSL RSA PKCS#1 v1.5 encrypt differential check because `openssl` is not available"); @@ -1709,7 +1774,7 @@ fn generated_private_key_outputs_verify_and_decrypt_with_external_oracles() { let mut pkcs1v15_signature = vec![0u8; key.signature_len()]; key .sign_pkcs1v15($pkcs1_profile, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("generated-key PKCS#1 v1.5 signing must succeed"); $assert_pkcs1(&public_pkcs1, message, &pkcs1v15_signature, true); assert_ring_cavp("pkcs1v15", $pss_sha, &public_pkcs1, message, &pkcs1v15_signature, true); assert_aws_lc_rs_cavp("pkcs1v15", $pss_sha, &public_pkcs1, message, &pkcs1v15_signature, true); @@ -1725,7 +1790,9 @@ fn generated_private_key_outputs_verify_and_decrypt_with_external_oracles() { assert_openssl_signature($digest_arg, &spki, message, &pkcs1v15_signature, &[], true); let mut pss_signature = vec![0u8; key.signature_len()]; - key.sign_pss($pss_profile, message, &mut pss_signature).unwrap(); + key + .sign_pss($pss_profile, message, &mut pss_signature) + .expect("generated-key PSS signing must succeed"); $assert_pss(&public_pkcs1, message, &pss_signature, true); assert_ring_cavp("pss", $pss_sha, &public_pkcs1, message, &pss_signature, true); assert_aws_lc_rs_cavp("pss", $pss_sha, &public_pkcs1, message, &pss_signature, true); @@ -1753,10 +1820,12 @@ fn generated_private_key_outputs_verify_and_decrypt_with_external_oracles() { &mut ciphertext, fill_rsa_random_from(&$seed), ) - .unwrap(); + .expect("generated-key OAEP encryption must succeed"); let rustcrypto_decrypting_key = RustCryptoOaepDecryptingKey::<$digest>::new_with_label(rustcrypto_from_pkcs1.clone(), label); - let rustcrypto_decrypted = rustcrypto_decrypting_key.decrypt(&ciphertext).unwrap(); + let rustcrypto_decrypted = rustcrypto_decrypting_key + .decrypt(&ciphertext) + .expect("RustCrypto must decrypt the matching generated-key OAEP ciphertext"); assert_eq!(rustcrypto_decrypted, plaintext); if let Some(openssl_decrypted) = openssl_oaep_decrypt(&pkcs1, &ciphertext, $oaep_digest) { @@ -1769,7 +1838,7 @@ fn generated_private_key_outputs_verify_and_decrypt_with_external_oracles() { let mut rscrypto_decrypted = vec![0u8; key.signature_len()]; let rscrypto_decrypted_len = key .decrypt_oaep($oaep_profile, b"", &openssl_ciphertext, &mut rscrypto_decrypted) - .unwrap(); + .expect("rscrypto must decrypt the matching OpenSSL OAEP ciphertext"); assert_eq!(&rscrypto_decrypted[..rscrypto_decrypted_len], plaintext); } else { eprintln!("skipping OpenSSL generated RSA OAEP encrypt check because `openssl` is not available"); @@ -1881,7 +1950,7 @@ fn generated_modern_private_key_outputs_verify_and_decrypt_with_external_oracles let mut pkcs1v15_signature = vec![0u8; key.signature_len()]; key .sign_pkcs1v15($pkcs1_profile, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("generated modern-key PKCS#1 v1.5 signing must succeed"); $assert_pkcs1(&public_pkcs1, message, &pkcs1v15_signature, true); assert_ring_cavp("pkcs1v15", $pss_sha, &public_pkcs1, message, &pkcs1v15_signature, true); assert_aws_lc_rs_cavp("pkcs1v15", $pss_sha, &public_pkcs1, message, &pkcs1v15_signature, true); @@ -1897,7 +1966,9 @@ fn generated_modern_private_key_outputs_verify_and_decrypt_with_external_oracles assert_openssl_signature($digest_arg, &spki, message, &pkcs1v15_signature, &[], true); let mut pss_signature = vec![0u8; key.signature_len()]; - key.sign_pss($pss_profile, message, &mut pss_signature).unwrap(); + key + .sign_pss($pss_profile, message, &mut pss_signature) + .expect("generated modern-key PSS signing must succeed"); $assert_pss(&public_pkcs1, message, &pss_signature, true); assert_ring_cavp("pss", $pss_sha, &public_pkcs1, message, &pss_signature, true); assert_aws_lc_rs_cavp("pss", $pss_sha, &public_pkcs1, message, &pss_signature, true); @@ -1925,10 +1996,12 @@ fn generated_modern_private_key_outputs_verify_and_decrypt_with_external_oracles &mut ciphertext, fill_rsa_random_from(&$seed), ) - .unwrap(); + .expect("generated modern-key OAEP encryption must succeed"); let rustcrypto_decrypting_key = RustCryptoOaepDecryptingKey::<$digest>::new_with_label(rustcrypto_from_pkcs1.clone(), label); - let rustcrypto_decrypted = rustcrypto_decrypting_key.decrypt(&ciphertext).unwrap(); + let rustcrypto_decrypted = rustcrypto_decrypting_key + .decrypt(&ciphertext) + .expect("RustCrypto must decrypt the matching generated modern-key OAEP ciphertext"); assert_eq!(rustcrypto_decrypted, plaintext); if let Some(openssl_decrypted) = openssl_oaep_decrypt(&pkcs1, &ciphertext, $oaep_digest) { @@ -1941,7 +2014,7 @@ fn generated_modern_private_key_outputs_verify_and_decrypt_with_external_oracles let mut rscrypto_decrypted = vec![0u8; key.signature_len()]; let rscrypto_decrypted_len = key .decrypt_oaep($oaep_profile, b"", &openssl_ciphertext, &mut rscrypto_decrypted) - .unwrap(); + .expect("rscrypto must decrypt the matching OpenSSL OAEP ciphertext"); assert_eq!(&rscrypto_decrypted[..rscrypto_decrypted_len], plaintext); } else { eprintln!("skipping OpenSSL generated RSA-{bits} OAEP encrypt check because `openssl` is not available"); @@ -2007,12 +2080,10 @@ fn x509_spki_public_key_preserves_pss_key_algorithm_constraints() { ); let pss_only_key = legacy_x509_public_key_from_spki(&pss_only_spki); assert_eq!(pss_only_key.key_algorithm(), RsaX509PublicKeyAlgorithm::RsaPss); - assert!( - pss_only_key - .key_algorithm() - .permits_signature_profile(RsaSignatureProfile::pss(RsaPssProfile::Sha256)) - .is_ok() - ); + pss_only_key + .key_algorithm() + .permits_signature_profile(RsaSignatureProfile::pss(RsaPssProfile::Sha256)) + .expect("an unrestricted PSS key must permit PSS SHA-256"); assert_eq!( pss_only_key .key_algorithm() @@ -2029,12 +2100,10 @@ fn x509_spki_public_key_preserves_pss_key_algorithm_constraints() { minimum_salt_len: 32, } ); - assert!( - restricted_key - .key_algorithm() - .permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 64)) - .is_ok() - ); + restricted_key + .key_algorithm() + .permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 64)) + .expect("a SHA-256 PSS key with a 32-byte minimum salt must permit a 64-byte salt"); assert_eq!( restricted_key .key_algorithm() @@ -2105,16 +2174,12 @@ fn x509_restricted_pss_key_enforces_rfc4055_signature_parameter_validation() { minimum_salt_len: 32, }; - assert!( - key_algorithm - .permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 32)) - .is_ok() - ); - assert!( - key_algorithm - .permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 64)) - .is_ok() - ); + key_algorithm + .permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 32)) + .expect("a restricted PSS key must permit its minimum salt length"); + key_algorithm + .permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 64)) + .expect("a restricted PSS key must permit a salt longer than its minimum"); assert_eq!( key_algorithm.permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 31)), Err(RsaProtocolAlgorithmError::UnsupportedAlgorithm) @@ -2184,11 +2249,9 @@ fn x509_spki_constraints_are_enforced_during_signature_verification() { let pss_algorithm = x509_pss_algorithm(RsaPssProfile::Sha256, 32, None); let pss_sig = pss_fixture_signature_sha256(); - assert!( - pss32_key - .verify_signature_from_x509_algorithm_der(&pss_algorithm, pss_fixture_message(), &pss_sig) - .is_ok() - ); + pss32_key + .verify_signature_from_x509_algorithm_der(&pss_algorithm, pss_fixture_message(), &pss_sig) + .expect("a restricted PSS key must verify a signature with matching parameters"); assert_eq!( pss32_key.verify_signature_from_x509_algorithm_der( &algorithm_identifier(SHA256_WITH_RSA_ENCRYPTION_OID, Some(&null())), @@ -2239,16 +2302,12 @@ fn x509_signature_verification_adapter_failures_are_opaque() { fn x509_certificate_signature_verification_accepts_real_rsa_certificates() { let issuer = legacy_x509_public_key_from_spki(&x509_certificate_fixture_public_key()); - assert!( - issuer - .verify_x509_certificate_signature_der(&x509_pkcs1v15_certificate_fixture()) - .is_ok() - ); - assert!( - issuer - .verify_x509_certificate_signature_der(&x509_pss_certificate_fixture()) - .is_ok() - ); + issuer + .verify_x509_certificate_signature_der(&x509_pkcs1v15_certificate_fixture()) + .expect("the RSA issuer must verify the PKCS#1 v1.5 certificate fixture"); + issuer + .verify_x509_certificate_signature_der(&x509_pss_certificate_fixture()) + .expect("the RSA issuer must verify the PSS certificate fixture"); let public_key = legacy_public_key_from_spki(&x509_certificate_fixture_public_key()); let pkcs1 = valid_pkcs1_with_modulus_and_exponent( @@ -2260,11 +2319,9 @@ fn x509_certificate_signature_verification_accepts_real_rsa_certificates() { &algorithm_identifier(ID_RSASSA_PSS_OID, None), )); - assert!( - pss_issuer - .verify_x509_certificate_signature_der(&x509_pss_certificate_fixture()) - .is_ok() - ); + pss_issuer + .verify_x509_certificate_signature_der(&x509_pss_certificate_fixture()) + .expect("the PSS issuer must verify the PSS certificate fixture"); assert_eq!( pss_issuer.verify_x509_certificate_signature_der(&x509_pkcs1v15_certificate_fixture()), Err(VerificationError::new()) @@ -2275,22 +2332,20 @@ fn x509_certificate_signature_verification_accepts_real_rsa_certificates() { fn x509_certificate_signature_algorithms_accept_absent_and_null_sha2_rsa_params() { let (null_params_spki, null_params_certificate) = x509_sha256_rsa_self_signed_certificate(Some(&null())); let null_params_issuer = legacy_x509_public_key_from_spki(&null_params_spki); - assert!( - null_params_issuer - .verify_x509_certificate_signature_der(&null_params_certificate) - .is_ok() - ); + null_params_issuer + .verify_x509_certificate_signature_der(&null_params_certificate) + .expect("SHA-256 RSA parameters encoded as NULL must verify"); let (absent_params_spki, absent_params_certificate) = x509_sha256_rsa_self_signed_certificate(None); let absent_params_issuer = legacy_x509_public_key_from_spki(&absent_params_spki); - assert!( - absent_params_issuer - .verify_x509_certificate_signature_der(&absent_params_certificate) - .is_ok() - ); + absent_params_issuer + .verify_x509_certificate_signature_der(&absent_params_certificate) + .expect("absent SHA-256 RSA parameters must verify"); let mut tampered_absent_params_certificate = absent_params_certificate; - *tampered_absent_params_certificate.last_mut().unwrap() ^= 0x01; + *tampered_absent_params_certificate + .last_mut() + .expect("the certificate fixture must contain a signature byte") ^= 0x01; assert_eq!( absent_params_issuer.verify_x509_certificate_signature_der(&tampered_absent_params_certificate), Err(VerificationError::new()) @@ -2339,8 +2394,9 @@ fn x509_certificate_chain_signature_fixtures_cover_rsa_pss_and_pkcs1v15() { pss_certificate.as_slice(), ), ] { - assert!( - issuer.verify_x509_certificate_signature_der(certificate).is_ok(), + assert_eq!( + issuer.verify_x509_certificate_signature_der(certificate), + Ok(()), "{name}" ); } @@ -2365,7 +2421,9 @@ fn x509_certificate_chain_signature_fixtures_cover_rsa_pss_and_pkcs1v15() { } let mut tampered_pss_certificate = pss_certificate; - *tampered_pss_certificate.last_mut().unwrap() ^= 0x01; + *tampered_pss_certificate + .last_mut() + .expect("the PSS certificate fixture must contain a signature byte") ^= 0x01; assert_eq!( rsae_issuer.verify_x509_certificate_signature_der(&tampered_pss_certificate), Err(VerificationError::new()) @@ -2377,7 +2435,9 @@ fn x509_certificate_signature_verification_rejects_malformed_and_confused_certif let issuer = legacy_x509_public_key_from_spki(&x509_certificate_fixture_public_key()); let mut tampered = x509_pkcs1v15_certificate_fixture(); - *tampered.last_mut().unwrap() ^= 0x01; + *tampered + .last_mut() + .expect("the PKCS#1 v1.5 certificate fixture must contain a signature byte") ^= 0x01; assert_eq!( issuer.verify_x509_certificate_signature_der(&tampered), Err(VerificationError::new()) @@ -2524,18 +2584,14 @@ fn tls_signature_scheme_advertisement_matches_executable_key_constraints() { for algorithm in [rsae, pss, pss_sha256_32, pss_sha256_64] { for scheme in algorithm.advertised_tls13_signature_schemes().iter() { - assert!( - algorithm.signature_profile_from_tls13_signature_scheme(scheme).is_ok(), - "advertised TLS 1.3 scheme {scheme:#06x} must be executable by {algorithm:?}" - ); + let _profile = algorithm + .signature_profile_from_tls13_signature_scheme(scheme) + .expect("every advertised TLS 1.3 scheme must map to an executable profile"); } for scheme in algorithm.advertised_tls_certificate_signature_schemes().iter() { - assert!( - algorithm - .signature_profile_from_tls_certificate_signature_scheme(scheme) - .is_ok(), - "advertised TLS certificate scheme {scheme:#06x} must be executable by {algorithm:?}" - ); + let _profile = algorithm + .signature_profile_from_tls_certificate_signature_scheme(scheme) + .expect("every advertised TLS certificate scheme must map to an executable profile"); } for unsupported_legacy_scheme in [0x0101, 0x0201, 0x0301, 0x0420, 0x0520, 0x0620] { @@ -2555,8 +2611,8 @@ fn tls_signature_scheme_advertisement_matches_executable_key_constraints() { #[test] fn tls_signature_scheme_advertisement_executes_real_verification_helpers() { - let rsae_key = RsaX509PublicKey::from_spki_der(RSA3072_SPKI).unwrap(); - let rsa3072_public = RsaPublicKey::from_spki_der(RSA3072_SPKI).unwrap(); + let rsae_key = RsaX509PublicKey::from_spki_der(RSA3072_SPKI).expect("the RSA-3072 X.509 fixture must parse"); + let rsa3072_public = RsaPublicKey::from_spki_der(RSA3072_SPKI).expect("the RSA-3072 SPKI fixture must parse"); let rsa3072_pkcs1 = valid_pkcs1_with_modulus_and_exponent( rsa3072_public.modulus(), &exponent_bytes(rsa3072_public.public_exponent().as_u64()), @@ -2565,12 +2621,12 @@ fn tls_signature_scheme_advertisement_executes_real_verification_helpers() { &rsa3072_pkcs1, &algorithm_identifier(ID_RSASSA_PSS_OID, None), )) - .unwrap(); + .expect("the unrestricted RSA-PSS SPKI fixture must parse"); let restricted_pss_key = RsaX509PublicKey::from_spki_der(&spki_for_pkcs1_with_algorithm( &rsa3072_pkcs1, &x509_pss_algorithm(RsaPssProfile::Sha256, 32, None), )) - .unwrap(); + .expect("the restricted RSA-PSS SPKI fixture must parse"); assert!( rsae_key @@ -2584,53 +2640,45 @@ fn tls_signature_scheme_advertisement_executes_real_verification_helpers() { .advertised_tls_certificate_signature_schemes() .contains(0x0401) ); - assert!( - rsae_key - .verify_expected_tls13_signature_scheme( - 0x0804, - 0x0804, - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - RSA3072_PSS_SHA256, - ) - .is_ok() - ); + rsae_key + .verify_expected_tls13_signature_scheme( + 0x0804, + 0x0804, + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + RSA3072_PSS_SHA256, + ) + .expect("the advertised RSAE TLS 1.3 PSS scheme must verify its fixture"); let mut rsae_scratch = rsae_key.public_key().public_scratch(); - assert!( - rsae_key - .verify_expected_tls13_signature_scheme_with_scratch( - 0x0804, - 0x0804, - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - RSA3072_PSS_SHA256, - &mut rsae_scratch, - ) - .is_ok() - ); - assert!( - rsae_key - .verify_expected_tls_certificate_signature_scheme( - 0x0401, - 0x0401, - RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), - pkcs1v15_fixture_message(), - RSA3072_PKCS1V15_SHA256, - ) - .is_ok() - ); - assert!( - rsae_key - .verify_expected_tls_certificate_signature_scheme_with_scratch( - 0x0401, - 0x0401, - RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), - pkcs1v15_fixture_message(), - RSA3072_PKCS1V15_SHA256, - &mut rsae_scratch, - ) - .is_ok() - ); + rsae_key + .verify_expected_tls13_signature_scheme_with_scratch( + 0x0804, + 0x0804, + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + RSA3072_PSS_SHA256, + &mut rsae_scratch, + ) + .expect("the advertised RSAE TLS 1.3 PSS scheme must verify with scratch reuse"); + rsae_key + .verify_expected_tls_certificate_signature_scheme( + 0x0401, + 0x0401, + RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), + pkcs1v15_fixture_message(), + RSA3072_PKCS1V15_SHA256, + ) + .expect("the advertised RSAE certificate PKCS#1 v1.5 scheme must verify its fixture"); + rsae_key + .verify_expected_tls_certificate_signature_scheme_with_scratch( + 0x0401, + 0x0401, + RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), + pkcs1v15_fixture_message(), + RSA3072_PKCS1V15_SHA256, + &mut rsae_scratch, + ) + .expect("the advertised RSAE certificate PKCS#1 v1.5 scheme must verify with scratch reuse"); for key in [&pss_key, &restricted_pss_key] { assert!( @@ -2645,53 +2693,45 @@ fn tls_signature_scheme_advertisement_executes_real_verification_helpers() { .advertised_tls_certificate_signature_schemes() .contains(0x0809) ); - assert!( - key - .verify_expected_tls13_signature_scheme( - 0x0809, - 0x0809, - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - RSA3072_PSS_SHA256, - ) - .is_ok() - ); + key + .verify_expected_tls13_signature_scheme( + 0x0809, + 0x0809, + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + RSA3072_PSS_SHA256, + ) + .expect("the advertised PSS TLS 1.3 scheme must verify its fixture"); let mut scratch = key.public_key().public_scratch(); - assert!( - key - .verify_expected_tls13_signature_scheme_with_scratch( - 0x0809, - 0x0809, - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - RSA3072_PSS_SHA256, - &mut scratch, - ) - .is_ok() - ); - assert!( - key - .verify_expected_tls_certificate_signature_scheme( - 0x0809, - 0x0809, - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - RSA3072_PSS_SHA256, - ) - .is_ok() - ); - assert!( - key - .verify_expected_tls_certificate_signature_scheme_with_scratch( - 0x0809, - 0x0809, - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - RSA3072_PSS_SHA256, - &mut scratch, - ) - .is_ok() - ); + key + .verify_expected_tls13_signature_scheme_with_scratch( + 0x0809, + 0x0809, + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + RSA3072_PSS_SHA256, + &mut scratch, + ) + .expect("the advertised PSS TLS 1.3 scheme must verify with scratch reuse"); + key + .verify_expected_tls_certificate_signature_scheme( + 0x0809, + 0x0809, + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + RSA3072_PSS_SHA256, + ) + .expect("the advertised PSS certificate scheme must verify its fixture"); + key + .verify_expected_tls_certificate_signature_scheme_with_scratch( + 0x0809, + 0x0809, + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + RSA3072_PSS_SHA256, + &mut scratch, + ) + .expect("the advertised PSS certificate scheme must verify with scratch reuse"); } } @@ -2708,26 +2748,18 @@ fn tls_signature_scheme_verification_rejects_key_algorithm_confusion() { let pss_signature = pss_fixture_signature_sha256(); let pss_profile = RsaSignatureProfile::pss(RsaPssProfile::Sha256); - assert!( - rsae_key - .verify_tls13_signature_scheme(0x0804, pss_fixture_message(), &pss_signature) - .is_ok() - ); - assert!( - rsae_key - .verify_expected_tls13_signature_scheme(0x0804, 0x0804, pss_profile, pss_fixture_message(), &pss_signature) - .is_ok() - ); - assert!( - pss_key - .verify_tls13_signature_scheme(0x0809, pss_fixture_message(), &pss_signature) - .is_ok() - ); - assert!( - pss_key - .verify_expected_tls13_signature_scheme(0x0809, 0x0809, pss_profile, pss_fixture_message(), &pss_signature) - .is_ok() - ); + rsae_key + .verify_tls13_signature_scheme(0x0804, pss_fixture_message(), &pss_signature) + .expect("an RSAE key must verify its matching TLS 1.3 PSS fixture"); + rsae_key + .verify_expected_tls13_signature_scheme(0x0804, 0x0804, pss_profile, pss_fixture_message(), &pss_signature) + .expect("an RSAE key must verify matching expected and received TLS 1.3 schemes"); + pss_key + .verify_tls13_signature_scheme(0x0809, pss_fixture_message(), &pss_signature) + .expect("a PSS key must verify its matching TLS 1.3 PSS fixture"); + pss_key + .verify_expected_tls13_signature_scheme(0x0809, 0x0809, pss_profile, pss_fixture_message(), &pss_signature) + .expect("a PSS key must verify matching expected and received TLS 1.3 schemes"); assert_eq!( rsae_key.verify_expected_tls13_signature_scheme(0x0804, 0x0809, pss_profile, pss_fixture_message(), &pss_signature), Err(VerificationError::new()) @@ -2765,22 +2797,18 @@ fn tls_signature_scheme_verification_rejects_key_algorithm_confusion() { let rsae_pkcs1_key = legacy_x509_public_key_from_spki(&pkcs1v15_fixture_public_key()); let pkcs1_signature = pkcs1v15_fixture_signature_sha256(); let pkcs1_profile = RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256); - assert!( - rsae_pkcs1_key - .verify_tls_certificate_signature_scheme(0x0401, pkcs1v15_fixture_message(), &pkcs1_signature) - .is_ok() - ); - assert!( - rsae_pkcs1_key - .verify_expected_tls_certificate_signature_scheme( - 0x0401, - 0x0401, - pkcs1_profile, - pkcs1v15_fixture_message(), - &pkcs1_signature, - ) - .is_ok() - ); + rsae_pkcs1_key + .verify_tls_certificate_signature_scheme(0x0401, pkcs1v15_fixture_message(), &pkcs1_signature) + .expect("an RSAE key must verify its matching TLS certificate PKCS#1 v1.5 fixture"); + rsae_pkcs1_key + .verify_expected_tls_certificate_signature_scheme( + 0x0401, + 0x0401, + pkcs1_profile, + pkcs1v15_fixture_message(), + &pkcs1_signature, + ) + .expect("an RSAE key must verify matching expected and received TLS certificate schemes"); assert_eq!( rsae_pkcs1_key.verify_expected_tls_certificate_signature_scheme( 0x0401, @@ -3056,7 +3084,9 @@ fn protocol_verification_helpers_collapse_adapter_failures_to_opaque_error() { let issuer = legacy_x509_public_key_from_spki(&x509_certificate_fixture_public_key()); let mut certificate_scratch = issuer.public_key().public_scratch(); let mut tampered_certificate = x509_pkcs1v15_certificate_fixture(); - *tampered_certificate.last_mut().unwrap() ^= 0x01; + *tampered_certificate + .last_mut() + .expect("the certificate fixture must contain a signature byte") ^= 0x01; let certificate_with_short_signature = x509_certificate( &minimal_tbs_certificate(&pkcs1_algorithm), &pkcs1_algorithm, @@ -3112,55 +3142,49 @@ fn rsa_policy_boundary_separates_legacy_rsa2048_from_modern_rsa3072_verification let legacy_policy = RsaPublicKeyPolicy::legacy_verification(); let modern_policy = RsaPublicKeyPolicy::modern_verification(); - let legacy_pss_key = RsaPublicKey::from_spki_der_with_policy(&pss_fixture_public_key(), &legacy_policy).unwrap(); + let legacy_pss_key = RsaPublicKey::from_spki_der_with_policy(&pss_fixture_public_key(), &legacy_policy) + .expect("the RSA-2048 PSS fixture must satisfy the legacy policy"); assert_eq!(legacy_pss_key.modulus_bits(), 2048); - assert!( - legacy_pss_key - .verify_pss( - RsaPssProfile::Sha256, - pss_fixture_message(), - &pss_fixture_signature_sha256() - ) - .is_ok() - ); + legacy_pss_key + .verify_pss( + RsaPssProfile::Sha256, + pss_fixture_message(), + &pss_fixture_signature_sha256(), + ) + .expect("the RSA-2048 PSS fixture must verify under the legacy policy"); assert_eq!( RsaPublicKey::from_spki_der_with_policy(&pss_fixture_public_key(), &modern_policy), Err(RsaKeyError::InvalidModulus) ); - let legacy_pkcs1v15_key = - RsaPublicKey::from_spki_der_with_policy(&pkcs1v15_fixture_public_key(), &legacy_policy).unwrap(); + let legacy_pkcs1v15_key = RsaPublicKey::from_spki_der_with_policy(&pkcs1v15_fixture_public_key(), &legacy_policy) + .expect("the RSA-2048 PKCS#1 v1.5 fixture must satisfy the legacy policy"); assert_eq!(legacy_pkcs1v15_key.modulus_bits(), 2048); - assert!( - legacy_pkcs1v15_key - .verify_pkcs1v15( - RsaPkcs1v15Profile::Sha256, - pkcs1v15_fixture_message(), - &pkcs1v15_fixture_signature_sha256(), - ) - .is_ok() - ); + legacy_pkcs1v15_key + .verify_pkcs1v15( + RsaPkcs1v15Profile::Sha256, + pkcs1v15_fixture_message(), + &pkcs1v15_fixture_signature_sha256(), + ) + .expect("the RSA-2048 PKCS#1 v1.5 fixture must verify under the legacy policy"); assert_eq!( RsaPublicKey::from_spki_der_with_policy(&pkcs1v15_fixture_public_key(), &modern_policy), Err(RsaKeyError::InvalidModulus) ); - let modern_key = RsaPublicKey::from_spki_der_with_policy(RSA3072_SPKI, &modern_policy).unwrap(); + let modern_key = RsaPublicKey::from_spki_der_with_policy(RSA3072_SPKI, &modern_policy) + .expect("the RSA-3072 fixture must satisfy the modern policy"); assert_eq!(modern_key.modulus_bits(), 3072); - assert!( - modern_key - .verify_pss(RsaPssProfile::Sha256, pss_fixture_message(), RSA3072_PSS_SHA256) - .is_ok() - ); - assert!( - modern_key - .verify_pkcs1v15( - RsaPkcs1v15Profile::Sha256, - pkcs1v15_fixture_message(), - RSA3072_PKCS1V15_SHA256 - ) - .is_ok() - ); + modern_key + .verify_pss(RsaPssProfile::Sha256, pss_fixture_message(), RSA3072_PSS_SHA256) + .expect("the RSA-3072 PSS fixture must verify under the modern policy"); + modern_key + .verify_pkcs1v15( + RsaPkcs1v15Profile::Sha256, + pkcs1v15_fixture_message(), + RSA3072_PKCS1V15_SHA256, + ) + .expect("the RSA-3072 PKCS#1 v1.5 fixture must verify under the modern policy"); } #[test] @@ -3391,7 +3415,8 @@ fn pkcs1_public_key_can_accept_legacy_small_fermat_exponents_by_policy() { let policy = RsaPublicKeyPolicy::legacy_verification().allow_legacy_small_exponents(); let der = pkcs1_with_parts(integer_unsigned(&rsa2048_modulus()), integer_unsigned(&[0x03])); - let key = RsaPublicKey::from_pkcs1_der_with_policy(&der, &policy).unwrap(); + let key = RsaPublicKey::from_pkcs1_der_with_policy(&der, &policy) + .expect("the legacy policy must accept the RSA exponent 3 fixture"); assert_eq!(key.public_exponent().as_u64(), 3); } @@ -3429,14 +3454,14 @@ fn public_operation_boundary_representatives_match_independent_reference_across_ let key = if name == "rsa2048" { legacy_public_key_from_spki(spki) } else { - RsaPublicKey::from_spki_der(spki).unwrap() + RsaPublicKey::from_spki_der(spki).expect("the modern RSA boundary fixture must parse") }; let len = key.modulus().len(); let mut out = vec![0u8; len]; let mut scratch = key.public_scratch(); let mut one = vec![0u8; len]; - *one.last_mut().unwrap() = 1; + *one.last_mut().expect("a supported RSA representative must be nonempty") = 1; let mut leading_zero = vec![0u8; len]; let leading_zero_tail = b"rscrypto-rsa-boundary-reference"; leading_zero[len - leading_zero_tail.len()..].copy_from_slice(leading_zero_tail); @@ -3450,7 +3475,7 @@ fn public_operation_boundary_representatives_match_independent_reference_across_ for (representative_name, representative) in valid_representatives { key .public_operation_with_scratch(&representative, &mut out, &mut scratch) - .unwrap_or_else(|error| panic!("{name} {representative_name} public operation failed: {error:?}")); + .expect("a valid boundary representative must complete the RSA public operation"); assert_eq!( out, rsa_public_operation_reference(&key, &representative), @@ -3526,7 +3551,7 @@ ec34e8c72cc58fd5324fbe1ddd9714909caedfaa38706cfa66d9bc1026ba3ec1188092392a54a\ key .public_operation_with_scratch(&input, &mut out, &mut scratch) - .unwrap(); + .expect("the independent RSA-65537 public-operation vector must be valid"); assert_eq!(out, expected); } @@ -3578,16 +3603,20 @@ fc87eb5a2ea7a141091f3d2fa37a581ef39e6e496e5e476d43b6", let key_e3 = RsaPublicKey::from_pkcs1_der_with_policy(&valid_pkcs1_with_modulus_and_exponent(&modulus, &[0x03]), &policy) - .unwrap(); + .expect("the legacy policy must accept the exponent 3 vector"); let key_e17 = RsaPublicKey::from_pkcs1_der_with_policy(&valid_pkcs1_with_modulus_and_exponent(&modulus, &[0x11]), &policy) - .unwrap(); + .expect("the legacy policy must accept the exponent 17 vector"); let mut out = vec![0u8; modulus.len()]; - key_e3.public_operation(&input, &mut out).unwrap(); + key_e3 + .public_operation(&input, &mut out) + .expect("the exponent 3 public-operation vector must be valid"); assert_eq!(out, expected_e3); - key_e17.public_operation(&input, &mut out).unwrap(); + key_e17 + .public_operation(&input, &mut out) + .expect("the exponent 17 public-operation vector must be valid"); assert_eq!(out, expected_e17); } @@ -3605,7 +3634,7 @@ fn public_scratch_reuses_after_modulus_minus_one_operation() { core::hint::black_box(&mut out), &mut scratch, ) - .unwrap(); + .expect("the modulus-minus-one public operation must succeed across scratch reuse"); } key .verify_pss_with_scratch( @@ -3614,7 +3643,7 @@ fn public_scratch_reuses_after_modulus_minus_one_operation() { &pss_fixture_signature_sha256(), &mut scratch, ) - .unwrap(); + .expect("PSS verification must succeed after repeated scratch reuse"); } #[cfg(feature = "diag")] @@ -3651,7 +3680,7 @@ d5add90a8a212c10dd997b0a4efcb3df990808509dcb28c504e0649827a83ffd864395d1f62f2\ ] { let key = RsaPublicKey::from_pkcs1_der_with_policy(&valid_pkcs1_with_modulus_and_exponent(&modulus, exponent), &policy) - .unwrap(); + .expect("the diagnostic exponent fixture must satisfy its policy"); let mut current = vec![0u8; modulus.len()]; let mut product = vec![0u8; modulus.len()]; let mut comba = vec![0u8; modulus.len()]; @@ -3664,11 +3693,15 @@ d5add90a8a212c10dd997b0a4efcb3df990808509dcb28c504e0649827a83ffd864395d1f62f2\ key .public_operation_with_scratch(&input, &mut current, &mut scratch) - .unwrap(); - diag_rsa_public_operation_product(&key, &input, &mut product, &mut product_scratch).unwrap(); - diag_rsa_public_operation_comba_product(&key, &input, &mut comba, &mut comba_scratch).unwrap(); - diag_rsa_public_operation_window2_exponent(&key, &input, &mut window2, &mut window2_scratch).unwrap(); - diag_rsa_public_operation_bitserial(&key, &input, &mut bitserial).unwrap(); + .expect("the current Montgomery path must accept the diagnostic fixture"); + diag_rsa_public_operation_product(&key, &input, &mut product, &mut product_scratch) + .expect("the product Montgomery candidate must accept the diagnostic fixture"); + diag_rsa_public_operation_comba_product(&key, &input, &mut comba, &mut comba_scratch) + .expect("the Comba Montgomery candidate must accept the diagnostic fixture"); + diag_rsa_public_operation_window2_exponent(&key, &input, &mut window2, &mut window2_scratch) + .expect("the window-2 exponent candidate must accept the diagnostic fixture"); + diag_rsa_public_operation_bitserial(&key, &input, &mut bitserial) + .expect("the bit-serial candidate must accept the diagnostic fixture"); assert_eq!( product, current, @@ -3705,7 +3738,7 @@ fn public_operation_montgomery_candidates_match_current_path() { let key = if name == "rsa2048" { legacy_public_key_from_spki(spki) } else { - RsaPublicKey::from_spki_der(spki).unwrap() + RsaPublicKey::from_spki_der(spki).expect("the modern RSA diagnostic fixture must parse") }; let representative = modulus_minus_one(&key); let mut current = vec![0u8; key.modulus().len()]; @@ -3721,12 +3754,15 @@ fn public_operation_montgomery_candidates_match_current_path() { key .public_operation_with_scratch(&representative, &mut current, &mut scratch) - .unwrap(); - diag_rsa_public_operation_cios(&key, &representative, &mut cios, &mut cios_scratch).unwrap(); + .expect("the current Montgomery path must accept modulus minus one"); + diag_rsa_public_operation_cios(&key, &representative, &mut cios, &mut cios_scratch) + .expect("the CIOS candidate must accept modulus minus one"); diag_rsa_public_operation_cios_portable(&key, &representative, &mut cios_portable, &mut cios_portable_scratch) - .unwrap(); - diag_rsa_public_operation_comba_product(&key, &representative, &mut comba, &mut comba_scratch).unwrap(); - diag_rsa_public_operation_product(&key, &representative, &mut product, &mut product_scratch).unwrap(); + .expect("the portable CIOS candidate must accept modulus minus one"); + diag_rsa_public_operation_comba_product(&key, &representative, &mut comba, &mut comba_scratch) + .expect("the Comba candidate must accept modulus minus one"); + diag_rsa_public_operation_product(&key, &representative, &mut product, &mut product_scratch) + .expect("the product candidate must accept modulus minus one"); assert_eq!(cios, current, "CIOS mismatch for modulus-minus-one representative"); assert_eq!( cios_portable, current, @@ -3743,11 +3779,15 @@ fn public_operation_montgomery_candidates_match_current_path() { key .public_operation_with_scratch(signature, &mut current, &mut scratch) - .unwrap(); - diag_rsa_public_operation_cios(&key, signature, &mut cios, &mut cios_scratch).unwrap(); - diag_rsa_public_operation_cios_portable(&key, signature, &mut cios_portable, &mut cios_portable_scratch).unwrap(); - diag_rsa_public_operation_comba_product(&key, signature, &mut comba, &mut comba_scratch).unwrap(); - diag_rsa_public_operation_product(&key, signature, &mut product, &mut product_scratch).unwrap(); + .expect("the current Montgomery path must accept the signature fixture"); + diag_rsa_public_operation_cios(&key, signature, &mut cios, &mut cios_scratch) + .expect("the CIOS candidate must accept the signature fixture"); + diag_rsa_public_operation_cios_portable(&key, signature, &mut cios_portable, &mut cios_portable_scratch) + .expect("the portable CIOS candidate must accept the signature fixture"); + diag_rsa_public_operation_comba_product(&key, signature, &mut comba, &mut comba_scratch) + .expect("the Comba candidate must accept the signature fixture"); + diag_rsa_public_operation_product(&key, signature, &mut product, &mut product_scratch) + .expect("the product candidate must accept the signature fixture"); assert_eq!(cios, current, "CIOS mismatch for fixture signature representative"); assert_eq!( cios_portable, current, @@ -3800,9 +3840,15 @@ aec5205b05ff989176db1199e8d34341380f501c34973526d024ef9fd87108e041c16625937a\ c2b691954f9bd86140e31acf6a8a2b9d28cba358e509dfc234c1e33e223c", ); - assert!(key.verify_pss(RsaPssProfile::Sha256, message, &sig256).is_ok()); - assert!(key.verify_pss(RsaPssProfile::Sha384, message, &sig384).is_ok()); - assert!(key.verify_pss(RsaPssProfile::Sha512, message, &sig512).is_ok()); + key + .verify_pss(RsaPssProfile::Sha256, message, &sig256) + .expect("the OpenSSL PSS SHA-256 vector must verify"); + key + .verify_pss(RsaPssProfile::Sha384, message, &sig384) + .expect("the OpenSSL PSS SHA-384 vector must verify"); + key + .verify_pss(RsaPssProfile::Sha512, message, &sig512) + .expect("the OpenSSL PSS SHA-512 vector must verify"); } #[test] @@ -3832,25 +3878,21 @@ fn typed_signature_profile_dispatches_and_rejects_algorithm_confusion() { let pss_key = legacy_public_key_from_spki(&pss_fixture_public_key()); let pss_sig = pss_fixture_signature_sha256(); let mut pss_scratch = pss_key.public_scratch(); - assert!( - pss_key - .verify_signature( - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - &pss_sig, - ) - .is_ok() - ); - assert!( - pss_key - .verify_signature_with_scratch( - RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 32), - pss_fixture_message(), - &pss_sig, - &mut pss_scratch, - ) - .is_ok() - ); + pss_key + .verify_signature( + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + &pss_sig, + ) + .expect("the typed PSS profile must verify its matching fixture"); + pss_key + .verify_signature_with_scratch( + RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 32), + pss_fixture_message(), + &pss_sig, + &mut pss_scratch, + ) + .expect("the typed PSS profile must verify with scratch reuse"); assert_eq!( pss_key.verify_signature( RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), @@ -3871,25 +3913,21 @@ fn typed_signature_profile_dispatches_and_rejects_algorithm_confusion() { let pkcs1_key = legacy_public_key_from_spki(&pkcs1v15_fixture_public_key()); let pkcs1_sig = pkcs1v15_fixture_signature_sha256(); let mut pkcs1_scratch = pkcs1_key.public_scratch(); - assert!( - pkcs1_key - .verify_signature( - RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), - pkcs1v15_fixture_message(), - &pkcs1_sig, - ) - .is_ok() - ); - assert!( - pkcs1_key - .verify_signature_with_scratch( - RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), - pkcs1v15_fixture_message(), - &pkcs1_sig, - &mut pkcs1_scratch, - ) - .is_ok() - ); + pkcs1_key + .verify_signature( + RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), + pkcs1v15_fixture_message(), + &pkcs1_sig, + ) + .expect("the typed PKCS#1 v1.5 profile must verify its matching fixture"); + pkcs1_key + .verify_signature_with_scratch( + RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), + pkcs1v15_fixture_message(), + &pkcs1_sig, + &mut pkcs1_scratch, + ) + .expect("the typed PKCS#1 v1.5 profile must verify with scratch reuse"); assert_eq!( pkcs1_key.verify_signature( RsaSignatureProfile::pss(RsaPssProfile::Sha256), @@ -4053,27 +4091,23 @@ fn provider_facing_legacy_algorithm_rejections_are_typed() { fn protocol_mapped_profiles_verify_and_reject_algorithm_confusion() { let pss_key = legacy_public_key_from_spki(&pss_fixture_public_key()); let pss_sig = pss_fixture_signature_sha256(); - assert!( - pss_key - .verify_signature( - RsaSignatureProfile::from_tls13_signature_scheme(0x0804).unwrap(), - pss_fixture_message(), - &pss_sig, - ) - .is_ok() - ); - assert!( - pss_key - .verify_signature( - RsaJwtAlgorithm::Ps256.signature_profile(), - pss_fixture_message(), - &pss_sig, - ) - .is_ok() - ); + pss_key + .verify_signature( + RsaSignatureProfile::from_tls13_signature_scheme(0x0804).expect("TLS scheme 0x0804 must map to RSA-PSS SHA-256"), + pss_fixture_message(), + &pss_sig, + ) + .expect("the TLS-mapped PSS profile must verify its matching fixture"); + pss_key + .verify_signature( + RsaJwtAlgorithm::Ps256.signature_profile(), + pss_fixture_message(), + &pss_sig, + ) + .expect("the JWT-mapped PSS profile must verify its matching fixture"); assert_eq!( pss_key.verify_signature( - RsaSignatureProfile::from_cose_algorithm_id(-257).unwrap(), + RsaSignatureProfile::from_cose_algorithm_id(-257).expect("COSE algorithm -257 must map to RS256"), pss_fixture_message(), &pss_sig, ), @@ -4082,27 +4116,24 @@ fn protocol_mapped_profiles_verify_and_reject_algorithm_confusion() { let pkcs1_key = legacy_public_key_from_spki(&pkcs1v15_fixture_public_key()); let pkcs1_sig = pkcs1v15_fixture_signature_sha256(); - assert!( - pkcs1_key - .verify_signature( - RsaSignatureProfile::from_tls_certificate_signature_scheme(0x0401).unwrap(), - pkcs1v15_fixture_message(), - &pkcs1_sig, - ) - .is_ok() - ); - assert!( - pkcs1_key - .verify_signature( - RsaJwtAlgorithm::Rs256.signature_profile(), - pkcs1v15_fixture_message(), - &pkcs1_sig, - ) - .is_ok() - ); + pkcs1_key + .verify_signature( + RsaSignatureProfile::from_tls_certificate_signature_scheme(0x0401) + .expect("TLS certificate scheme 0x0401 must map to PKCS#1 v1.5 SHA-256"), + pkcs1v15_fixture_message(), + &pkcs1_sig, + ) + .expect("the TLS-mapped PKCS#1 v1.5 profile must verify its matching fixture"); + pkcs1_key + .verify_signature( + RsaJwtAlgorithm::Rs256.signature_profile(), + pkcs1v15_fixture_message(), + &pkcs1_sig, + ) + .expect("the JWT-mapped PKCS#1 v1.5 profile must verify its matching fixture"); assert_eq!( pkcs1_key.verify_signature( - RsaSignatureProfile::from_cose_algorithm_id(-37).unwrap(), + RsaSignatureProfile::from_cose_algorithm_id(-37).expect("COSE algorithm -37 must map to PS256"), pkcs1v15_fixture_message(), &pkcs1_sig, ), @@ -4120,13 +4151,13 @@ fn jwt_policy_and_cose_helpers_reject_algorithm_confusion() { let pss_jwt = pss_key.jwt_verifier(RsaJwtAlgorithm::Ps256); assert_eq!(pss_jwt.algorithm(), RsaJwtAlgorithm::Ps256); - assert!(core::ptr::eq(pss_jwt.key(), &pss_key)); - assert!(pss_jwt.verify("PS256", pss_fixture_message(), &pss_sig).is_ok()); - assert!( - pss_jwt - .verify_with_scratch("PS256", pss_fixture_message(), &pss_sig, &mut scratch) - .is_ok() - ); + assert!(core::ptr::eq(pss_jwt.key(), &raw const pss_key)); + pss_jwt + .verify("PS256", pss_fixture_message(), &pss_sig) + .expect("the PS256 JWT verifier must accept its matching fixture"); + pss_jwt + .verify_with_scratch("PS256", pss_fixture_message(), &pss_sig, &mut scratch) + .expect("the PS256 JWT verifier must accept its matching fixture with scratch reuse"); for header_alg in [ "", "none", @@ -4147,23 +4178,19 @@ fn jwt_policy_and_cose_helpers_reject_algorithm_confusion() { "accepted noncanonical or mismatched JWT alg {header_alg:?}", ); } - assert!( - pss_key - .verify_expected_cose_algorithm_id(-37, -37, pss_profile, pss_fixture_message(), &pss_sig) - .is_ok() - ); - assert!( - pss_key - .verify_expected_cose_algorithm_id_with_scratch( - -37, - -37, - pss_profile, - pss_fixture_message(), - &pss_sig, - &mut scratch, - ) - .is_ok() - ); + pss_key + .verify_expected_cose_algorithm_id(-37, -37, pss_profile, pss_fixture_message(), &pss_sig) + .expect("matching PS256 COSE identifiers must verify"); + pss_key + .verify_expected_cose_algorithm_id_with_scratch( + -37, + -37, + pss_profile, + pss_fixture_message(), + &pss_sig, + &mut scratch, + ) + .expect("matching PS256 COSE identifiers must verify with scratch reuse"); for result in [ pss_key .jwt_verifier(RsaJwtAlgorithm::Rs256) @@ -4181,33 +4208,25 @@ fn jwt_policy_and_cose_helpers_reject_algorithm_confusion() { let mut scratch = pkcs1_key.public_scratch(); let pkcs1_jwt = pkcs1_key.jwt_verifier(RsaJwtAlgorithm::Rs256); - assert!( - pkcs1_jwt - .verify("RS256", pkcs1v15_fixture_message(), &pkcs1_sig) - .is_ok() - ); - assert!( - pkcs1_jwt - .verify_with_scratch("RS256", pkcs1v15_fixture_message(), &pkcs1_sig, &mut scratch) - .is_ok() - ); - assert!( - pkcs1_key - .verify_expected_cose_algorithm_id(-257, -257, pkcs1_profile, pkcs1v15_fixture_message(), &pkcs1_sig) - .is_ok() - ); - assert!( - pkcs1_key - .verify_expected_cose_algorithm_id_with_scratch( - -257, - -257, - pkcs1_profile, - pkcs1v15_fixture_message(), - &pkcs1_sig, - &mut scratch, - ) - .is_ok() - ); + pkcs1_jwt + .verify("RS256", pkcs1v15_fixture_message(), &pkcs1_sig) + .expect("the RS256 JWT verifier must accept its matching fixture"); + pkcs1_jwt + .verify_with_scratch("RS256", pkcs1v15_fixture_message(), &pkcs1_sig, &mut scratch) + .expect("the RS256 JWT verifier must accept its matching fixture with scratch reuse"); + pkcs1_key + .verify_expected_cose_algorithm_id(-257, -257, pkcs1_profile, pkcs1v15_fixture_message(), &pkcs1_sig) + .expect("matching RS256 COSE identifiers must verify"); + pkcs1_key + .verify_expected_cose_algorithm_id_with_scratch( + -257, + -257, + pkcs1_profile, + pkcs1v15_fixture_message(), + &pkcs1_sig, + &mut scratch, + ) + .expect("matching RS256 COSE identifiers must verify with scratch reuse"); for result in [ pkcs1_key .jwt_verifier(RsaJwtAlgorithm::Ps256) @@ -4353,16 +4372,14 @@ fn x509_signature_algorithm_mapping_is_strict_and_rejects_sha1_defaults() { fn x509_mapped_profiles_verify_real_fixtures_and_reject_padding_mismatch() { let pss_key = legacy_public_key_from_spki(&pss_fixture_public_key()); let pss_sig = pss_fixture_signature_sha256(); - assert!( - pss_key - .verify_signature( - RsaSignatureProfile::from_x509_signature_algorithm_der(&x509_pss_algorithm(RsaPssProfile::Sha256, 32, None)) - .unwrap(), - pss_fixture_message(), - &pss_sig, - ) - .is_ok() - ); + pss_key + .verify_signature( + RsaSignatureProfile::from_x509_signature_algorithm_der(&x509_pss_algorithm(RsaPssProfile::Sha256, 32, None)) + .expect("the X.509 PSS SHA-256 algorithm fixture must map to a signature profile"), + pss_fixture_message(), + &pss_sig, + ) + .expect("the X.509-mapped PSS profile must verify its matching fixture"); let pkcs1_key = legacy_public_key_from_spki(&pkcs1v15_fixture_public_key()); let pkcs1_sig = pkcs1v15_fixture_signature_sha256(); @@ -4370,22 +4387,18 @@ fn x509_mapped_profiles_verify_real_fixtures_and_reject_padding_mismatch() { SHA256_WITH_RSA_ENCRYPTION_OID, Some(&null()), )) - .unwrap(); - assert!( - pkcs1_key - .verify_signature(pkcs1_profile, pkcs1v15_fixture_message(), &pkcs1_sig) - .is_ok() - ); + .expect("the X.509 PKCS#1 v1.5 SHA-256 algorithm fixture must map to a signature profile"); + pkcs1_key + .verify_signature(pkcs1_profile, pkcs1v15_fixture_message(), &pkcs1_sig) + .expect("the X.509-mapped PKCS#1 v1.5 profile must verify its matching fixture"); let pkcs1_x509_key = legacy_x509_public_key_from_spki(&pkcs1v15_fixture_public_key()); - assert!( - pkcs1_x509_key - .verify_signature_from_x509_algorithm_der( - &algorithm_identifier(SHA256_WITH_RSA_ENCRYPTION_OID, None), - pkcs1v15_fixture_message(), - &pkcs1_sig, - ) - .is_ok() - ); + pkcs1_x509_key + .verify_signature_from_x509_algorithm_der( + &algorithm_identifier(SHA256_WITH_RSA_ENCRYPTION_OID, None), + pkcs1v15_fixture_message(), + &pkcs1_sig, + ) + .expect("an X.509 PKCS#1 v1.5 algorithm with absent parameters must verify"); assert_eq!( pss_key.verify_signature(pkcs1_profile, pss_fixture_message(), &pss_sig), Err(VerificationError::new()) @@ -4428,21 +4441,15 @@ addc98f3b49437df55aad5e96a6b7db196f9d30de7173dda79944f51fc7a339655cd2727d47\ 4630a322103c344bc4c65add2214f60155b3819869210f19730544989fed6921bf", ); - assert!( - key - .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &sig256) - .is_ok() - ); - assert!( - key - .verify_pkcs1v15(RsaPkcs1v15Profile::Sha384, message, &sig384) - .is_ok() - ); - assert!( - key - .verify_pkcs1v15(RsaPkcs1v15Profile::Sha512, message, &sig512) - .is_ok() - ); + key + .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &sig256) + .expect("the OpenSSL PKCS#1 v1.5 SHA-256 vector must verify"); + key + .verify_pkcs1v15(RsaPkcs1v15Profile::Sha384, message, &sig384) + .expect("the OpenSSL PKCS#1 v1.5 SHA-384 vector must verify"); + key + .verify_pkcs1v15(RsaPkcs1v15Profile::Sha512, message, &sig512) + .expect("the OpenSSL PKCS#1 v1.5 SHA-512 vector must verify"); } #[test] @@ -4478,13 +4485,16 @@ fn pss_encoded_message_oracle_failures_are_opaque() { let mut encoded = vec![0u8; key.modulus().len()]; key .public_operation(&pss_fixture_signature_sha256(), &mut encoded) - .unwrap(); + .expect("the PSS signature fixture must decode to its encoded message"); let em_bits = key.modulus_bits().strict_sub(1); - assert!(diag_rsa_verify_pss_encoded(RsaPssProfile::Sha256, pss_fixture_message(), &encoded, em_bits).is_ok()); + diag_rsa_verify_pss_encoded(RsaPssProfile::Sha256, pss_fixture_message(), &encoded, em_bits) + .expect("the valid PSS encoded message must verify"); let mut bad_trailer = encoded.clone(); - *bad_trailer.last_mut().unwrap() ^= 0x01; + *bad_trailer + .last_mut() + .expect("the PSS encoded message must contain a trailer byte") ^= 0x01; assert_opaque_verification_failure(diag_rsa_verify_pss_encoded( RsaPssProfile::Sha256, pss_fixture_message(), @@ -4555,9 +4565,10 @@ fn pkcs1v15_encoded_message_oracle_failures_are_opaque() { let mut encoded = vec![0u8; key.modulus().len()]; key .public_operation(&pkcs1v15_fixture_signature_sha256(), &mut encoded) - .unwrap(); + .expect("the PKCS#1 v1.5 signature fixture must decode to its encoded message"); - assert!(diag_rsa_verify_pkcs1v15_encoded(RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), &encoded).is_ok()); + diag_rsa_verify_pkcs1v15_encoded(RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), &encoded) + .expect("the valid PKCS#1 v1.5 encoded message must verify"); let mut bad_prefix = encoded.clone(); bad_prefix[0] = 0x01; @@ -4602,7 +4613,9 @@ fn pkcs1v15_encoded_message_oracle_failures_are_opaque() { )); let mut bad_digest = encoded.clone(); - *bad_digest.last_mut().unwrap() ^= 0x80; + *bad_digest + .last_mut() + .expect("the PKCS#1 v1.5 encoded message must contain a digest byte") ^= 0x80; assert_opaque_verification_failure(diag_rsa_verify_pkcs1v15_encoded( RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), @@ -4629,11 +4642,9 @@ fn sha256_signature_verification_matches_external_oracles_for_valid_and_tampered let pss_pkcs1 = valid_pkcs1_with_modulus_and_exponent(pss_key.modulus(), &exponent_bytes(pss_key.public_exponent().as_u64())); - assert!( - pss_key - .verify_pss(RsaPssProfile::Sha256, pss_fixture_message(), &pss_sig) - .is_ok() - ); + pss_key + .verify_pss(RsaPssProfile::Sha256, pss_fixture_message(), &pss_sig) + .expect("the valid PSS SHA-256 fixture must verify before external-oracle comparison"); assert_ring_pss_sha256(&pss_pkcs1, pss_fixture_message(), &pss_sig, true); assert_aws_lc_rs_pss_sha256(&pss_pkcs1, pss_fixture_message(), &pss_sig, true); assert_rustcrypto_pss_sha256(&pss_pkcs1, pss_fixture_message(), &pss_sig, true); @@ -4645,7 +4656,7 @@ fn sha256_signature_verification_matches_external_oracles_for_valid_and_tampered true, ); - let mut tampered = pss_sig.clone(); + let mut tampered = pss_sig; tampered[0] ^= 0x80; assert!( pss_key @@ -4671,17 +4682,15 @@ fn sha256_signature_verification_matches_external_oracles_for_valid_and_tampered &exponent_bytes(pkcs1_key.public_exponent().as_u64()), ); - assert!( - pkcs1_key - .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), &pkcs1_sig) - .is_ok() - ); + pkcs1_key + .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), &pkcs1_sig) + .expect("the valid PKCS#1 v1.5 SHA-256 fixture must verify before external-oracle comparison"); assert_ring_pkcs1v15_sha256(&pkcs1_der, pkcs1v15_fixture_message(), &pkcs1_sig, true); assert_aws_lc_rs_pkcs1v15_sha256(&pkcs1_der, pkcs1v15_fixture_message(), &pkcs1_sig, true); assert_rustcrypto_pkcs1v15_sha256(&pkcs1_der, pkcs1v15_fixture_message(), &pkcs1_sig, true); assert_openssl_sha256(&pkcs1_spki, pkcs1v15_fixture_message(), &pkcs1_sig, &[], true); - let mut tampered = pkcs1_sig.clone(); + let mut tampered = pkcs1_sig; tampered[0] ^= 0x80; assert!( pkcs1_key @@ -4703,13 +4712,16 @@ fn nist_cavp_sha2_signatures_match_rust_external_oracles() { for test in tests { let scheme = cavp_field(test, "scheme"); let sha = cavp_field(test, "sha"); - let salt_len = - (scheme == "pss").then(|| test["salt_len"].as_u64().expect("CAVP PSS salt length must be numeric") as usize); + let salt_len = (scheme == "pss").then(|| { + usize::try_from(test["salt_len"].as_u64().expect("CAVP PSS salt length must be numeric")) + .expect("a CAVP PSS salt length must fit in usize") + }); let expected = match cavp_field(test, "result") { - "P" => true, - "F" => false, - other => panic!("unsupported CAVP result `{other}`"), - }; + "P" => Some(true), + "F" => Some(false), + _ => None, + } + .expect("CAVP result fields must be `P` or `F`"); let modulus = cavp_hex_to_vec(cavp_field(test, "n")); let exponent = cavp_hex_to_vec(cavp_field(test, "e")); @@ -4745,13 +4757,16 @@ fn nist_cavp_sha2_signatures_match_openssl_cli_when_available() { for test in tests { let scheme = cavp_field(test, "scheme"); let sha = cavp_field(test, "sha"); - let salt_len = - (scheme == "pss").then(|| test["salt_len"].as_u64().expect("CAVP PSS salt length must be numeric") as usize); + let salt_len = (scheme == "pss").then(|| { + usize::try_from(test["salt_len"].as_u64().expect("CAVP PSS salt length must be numeric")) + .expect("a CAVP PSS salt length must fit in usize") + }); let expected = match cavp_field(test, "result") { - "P" => true, - "F" => false, - other => panic!("unsupported CAVP result `{other}`"), - }; + "P" => Some(true), + "F" => Some(false), + _ => None, + } + .expect("CAVP result fields must be `P` or `F`"); let pkcs1 = valid_pkcs1_with_modulus_and_exponent( &cavp_hex_to_vec(cavp_field(test, "n")), &cavp_hex_to_vec(cavp_field(test, "e")), @@ -4760,14 +4775,15 @@ fn nist_cavp_sha2_signatures_match_openssl_cli_when_available() { let message = cavp_hex_to_vec(cavp_field(test, "msg")); let signature = cavp_hex_to_vec(cavp_field(test, "sig")); let digest_arg = match sha { - "SHA256" => "-sha256", - "SHA384" => "-sha384", - "SHA512" => "-sha512", - other => panic!("unsupported OpenSSL CAVP hash `{other}`"), - }; + "SHA256" => Some("-sha256"), + "SHA384" => Some("-sha384"), + "SHA512" => Some("-sha512"), + _ => None, + } + .expect("CAVP fixtures must use an OpenSSL-supported SHA-2 profile"); let pss_sigopts; let sigopts = if scheme == "pss" { - pss_sigopts = openssl_pss_sigopts(sha, salt_len.unwrap()); + pss_sigopts = openssl_pss_sigopts(sha, salt_len.expect("a CAVP PSS case must provide a salt length")); &pss_sigopts[..] } else { &[] @@ -4795,19 +4811,15 @@ fn generated_rsa_size_fixtures_verify_for_benchmark_matrix() { (4096, RSA4096_SPKI, RSA4096_PSS_SHA256, RSA4096_PKCS1V15_SHA256), (8192, RSA8192_SPKI, RSA8192_PSS_SHA256, RSA8192_PKCS1V15_SHA256), ] { - let key = RsaPublicKey::from_spki_der(spki).unwrap(); + let key = RsaPublicKey::from_spki_der(spki).expect("the generated RSA benchmark fixture must parse"); assert_eq!(key.modulus_bits(), bits); - assert!( - key - .verify_pss(RsaPssProfile::Sha256, pss_fixture_message(), pss_sig) - .is_ok() - ); - assert!( - key - .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), pkcs1_sig) - .is_ok() - ); + key + .verify_pss(RsaPssProfile::Sha256, pss_fixture_message(), pss_sig) + .expect("the generated RSA benchmark PSS fixture must verify"); + key + .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), pkcs1_sig) + .expect("the generated RSA benchmark PKCS#1 v1.5 fixture must verify"); let pkcs1_der = valid_pkcs1_with_modulus_and_exponent(key.modulus(), &exponent_bytes(key.public_exponent().as_u64())); @@ -4841,35 +4853,29 @@ fn aarch64_linux_dispatch_verifies_generated_rsa_fixtures() { (4096, RSA4096_SPKI, RSA4096_PSS_SHA256, RSA4096_PKCS1V15_SHA256), (8192, RSA8192_SPKI, RSA8192_PSS_SHA256, RSA8192_PKCS1V15_SHA256), ] { - let key = RsaPublicKey::from_spki_der(spki).unwrap(); + let key = RsaPublicKey::from_spki_der(spki).expect("the generated AArch64 Linux RSA fixture must parse"); let mut scratch = key.public_scratch(); assert_eq!(key.modulus_bits(), bits); - assert!( - key - .verify_pss_with_scratch(RsaPssProfile::Sha256, pss_fixture_message(), pss_sig, &mut scratch) - .is_ok(), - "RSA-{bits} PSS fixture failed through AArch64 Linux dispatch" - ); - assert!( - key - .verify_pkcs1v15_with_scratch( - RsaPkcs1v15Profile::Sha256, - pkcs1v15_fixture_message(), - pkcs1_sig, - &mut scratch, - ) - .is_ok(), - "RSA-{bits} PKCS1v15 fixture failed through AArch64 Linux dispatch" - ); + key + .verify_pss_with_scratch(RsaPssProfile::Sha256, pss_fixture_message(), pss_sig, &mut scratch) + .expect("the PSS fixture must verify through AArch64 Linux dispatch"); + key + .verify_pkcs1v15_with_scratch( + RsaPkcs1v15Profile::Sha256, + pkcs1v15_fixture_message(), + pkcs1_sig, + &mut scratch, + ) + .expect("the PKCS#1 v1.5 fixture must verify through AArch64 Linux dispatch"); } } proptest! { #[test] fn arbitrary_der_inputs_do_not_panic(input in proptest::collection::vec(any::(), 0..4096)) { - let _ = RsaPublicKey::from_pkcs1_der(&input); - let _ = RsaPublicKey::from_spki_der(&input); + let _pkcs1_result = RsaPublicKey::from_pkcs1_der(&input); + let _spki_result = RsaPublicKey::from_spki_der(&input); } #[test] @@ -4880,12 +4886,18 @@ proptest! { let key = arbitrary_verify_key(); let mut scratch = key.public_scratch(); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha256, &message, &signature, &mut scratch); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha384, &message, &signature, &mut scratch); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha512, &message, &signature, &mut scratch); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, &message, &signature, &mut scratch); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha384, &message, &signature, &mut scratch); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha512, &message, &signature, &mut scratch); + let _pss_sha256_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha256, &message, &signature, &mut scratch); + let _pss_sha384_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha384, &message, &signature, &mut scratch); + let _pss_sha512_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha512, &message, &signature, &mut scratch); + let _pkcs1_sha256_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, &message, &signature, &mut scratch); + let _pkcs1_sha384_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha384, &message, &signature, &mut scratch); + let _pkcs1_sha512_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha512, &message, &signature, &mut scratch); } #[test] @@ -4897,11 +4909,17 @@ proptest! { let signature = fixed_width_signature_candidate(&material, key.modulus().len()); let mut scratch = key.public_scratch(); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha256, &message, &signature, &mut scratch); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha384, &message, &signature, &mut scratch); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha512, &message, &signature, &mut scratch); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, &message, &signature, &mut scratch); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha384, &message, &signature, &mut scratch); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha512, &message, &signature, &mut scratch); + let _pss_sha256_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha256, &message, &signature, &mut scratch); + let _pss_sha384_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha384, &message, &signature, &mut scratch); + let _pss_sha512_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha512, &message, &signature, &mut scratch); + let _pkcs1_sha256_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, &message, &signature, &mut scratch); + let _pkcs1_sha384_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha384, &message, &signature, &mut scratch); + let _pkcs1_sha512_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha512, &message, &signature, &mut scratch); } } diff --git a/tests/rsa_wycheproof.rs b/tests/rsa_wycheproof.rs index 5daded75..3b8ed77d 100644 --- a/tests/rsa_wycheproof.rs +++ b/tests/rsa_wycheproof.rs @@ -2,7 +2,9 @@ use core::ops::Range; -use rscrypto::{RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKey, RsaPublicKeyPolicy}; +use rscrypto::{ + RsaBlindingPair, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKey, RsaPublicKeyPolicy, +}; use serde_json::Value; const PKCS1_SHA256: &str = include_str!("../testdata/rsa/wycheproof/rsa_signature_2048_sha256_test.json"); @@ -55,21 +57,30 @@ struct ExpectedCounts { invalid: usize, } +#[derive(Clone, Copy)] +enum WycheproofResult { + Valid, + Acceptable, + Invalid, +} + fn hex_to_vec(hex: &str) -> Vec { assert_eq!(hex.len() % 2, 0); let mut out = Vec::with_capacity(hex.len() / 2); for chunk in hex.as_bytes().chunks_exact(2) { - out.push((hex_value(chunk[0]) << 4) | hex_value(chunk[1])); + let high = hex_value(chunk[0]).expect("Wycheproof fixture must contain hexadecimal digits"); + let low = hex_value(chunk[1]).expect("Wycheproof fixture must contain hexadecimal digits"); + out.push((high << 4) | low); } out } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } @@ -87,7 +98,17 @@ fn test_cases(group: &Value) -> &[Value] { fn field<'a>(value: &'a Value, name: &str) -> &'a str { value[name] .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + .expect("Wycheproof fixture must contain the requested string field") +} + +fn wycheproof_result(result: &str) -> WycheproofResult { + match result { + "valid" => Some(WycheproofResult::Valid), + "acceptable" => Some(WycheproofResult::Acceptable), + "invalid" => Some(WycheproofResult::Invalid), + _ => None, + } + .expect("Wycheproof result must be valid, acceptable, or invalid") } fn public_key_from_wycheproof_spki(der: &[u8], key_size: u64, context: &str) -> RsaPublicKey { @@ -133,8 +154,8 @@ fn assert_pkcs1v15_wycheproof_vectors( .verify_pkcs1v15_with_scratch(profile, &msg, &sig, &mut scratch) .is_ok(); - match field(test, "result") { - "valid" => { + match wycheproof_result(field(test, "result")) { + WycheproofResult::Valid => { valid = valid.strict_add(1); assert!( verified, @@ -142,7 +163,7 @@ fn assert_pkcs1v15_wycheproof_vectors( test["tcId"] ); } - "invalid" => { + WycheproofResult::Invalid => { invalid = invalid.strict_add(1); assert!( !verified, @@ -151,10 +172,9 @@ fn assert_pkcs1v15_wycheproof_vectors( field(test, "comment") ); } - "acceptable" => { + WycheproofResult::Acceptable => { acceptable = acceptable.strict_add(1); } - other => panic!("unknown Wycheproof result `{other}`"), } } } @@ -199,8 +219,8 @@ fn assert_pss_wycheproof_vectors( .verify_pss_with_salt_len_and_scratch(profile, expected_salt_len, &msg, &sig, &mut scratch) .is_ok(); - match field(test, "result") { - "valid" => { + match wycheproof_result(field(test, "result")) { + WycheproofResult::Valid => { valid = valid.strict_add(1); assert!( verified, @@ -208,7 +228,7 @@ fn assert_pss_wycheproof_vectors( test["tcId"] ); } - "invalid" => { + WycheproofResult::Invalid => { invalid = invalid.strict_add(1); assert!( !verified, @@ -217,10 +237,9 @@ fn assert_pss_wycheproof_vectors( field(test, "comment") ); } - "acceptable" => { + WycheproofResult::Acceptable => { acceptable = acceptable.strict_add(1); } - other => panic!("unknown Wycheproof result `{other}`"), } } } @@ -295,20 +314,14 @@ fn assert_oaep_wycheproof_vectors( profile, &label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut plaintext, ); - match field(test, "result") { - "valid" => { + match wycheproof_result(field(test, "result")) { + WycheproofResult::Valid => { valid = valid.strict_add(1); - let plaintext_len = decrypted.unwrap_or_else(|error| { - panic!( - "Wycheproof OAEP tcId {} rejected valid ciphertext: {error}", - test["tcId"] - ) - }); + let plaintext_len = decrypted.expect("Wycheproof valid OAEP ciphertext must decrypt"); assert_eq!( &plaintext[..plaintext_len], hex_to_vec(field(test, "msg")).as_slice(), @@ -322,8 +335,7 @@ fn assert_oaep_wycheproof_vectors( profile, &label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_plaintext, &mut scratch, ) @@ -337,7 +349,7 @@ fn assert_oaep_wycheproof_vectors( scratch_valid_checked = true; } } - "invalid" => { + WycheproofResult::Invalid => { invalid = invalid.strict_add(1); assert!( decrypted.is_err(), @@ -358,8 +370,7 @@ fn assert_oaep_wycheproof_vectors( profile, &label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_plaintext, &mut scratch, ) @@ -376,10 +387,9 @@ fn assert_oaep_wycheproof_vectors( scratch_invalid_checked = true; } } - "acceptable" => { + WycheproofResult::Acceptable => { acceptable = acceptable.strict_add(1); } - other => panic!("unknown Wycheproof result `{other}`"), } } if group_has_valid { @@ -447,8 +457,7 @@ fn assert_oaep_mgf1sha1_vectors_are_rejected( profile, &label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut plaintext, ); let mut scratch_plaintext = vec![0xa5; key.public_key().modulus().len()]; @@ -456,14 +465,13 @@ fn assert_oaep_mgf1sha1_vectors_are_rejected( profile, &label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_plaintext, &mut scratch, ); - match field(test, "result") { - "valid" => { + match wycheproof_result(field(test, "result")) { + WycheproofResult::Valid => { valid = valid.strict_add(1); assert!( decrypted.is_err(), @@ -476,7 +484,7 @@ fn assert_oaep_mgf1sha1_vectors_are_rejected( test["tcId"] ); } - "invalid" => { + WycheproofResult::Invalid => { invalid = invalid.strict_add(1); assert!( decrypted.is_err(), @@ -491,7 +499,7 @@ fn assert_oaep_mgf1sha1_vectors_are_rejected( field(test, "comment") ); } - "acceptable" => { + WycheproofResult::Acceptable => { acceptable = acceptable.strict_add(1); assert!( decrypted.is_err(), @@ -504,7 +512,6 @@ fn assert_oaep_mgf1sha1_vectors_are_rejected( test["tcId"] ); } - other => panic!("unknown Wycheproof result `{other}`"), } assert!( plaintext.iter().all(|&byte| byte == 0), @@ -583,20 +590,14 @@ fn assert_rsaes_pkcs1v15_wycheproof_vectors( let mut plaintext = vec![0xa5; key.public_key().modulus().len()]; let decrypted = key.decrypt_pkcs1v15_with_blinding_factor( &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut plaintext, ); - match field(test, "result") { - "valid" => { + match wycheproof_result(field(test, "result")) { + WycheproofResult::Valid => { valid = valid.strict_add(1); - let plaintext_len = decrypted.unwrap_or_else(|error| { - panic!( - "Wycheproof RSAES-PKCS1-v1_5 tcId {} rejected valid ciphertext: {error}", - test["tcId"] - ) - }); + let plaintext_len = decrypted.expect("Wycheproof valid RSAES-PKCS1-v1_5 ciphertext must decrypt"); assert_eq!( &plaintext[..plaintext_len], hex_to_vec(field(test, "msg")).as_slice(), @@ -608,8 +609,7 @@ fn assert_rsaes_pkcs1v15_wycheproof_vectors( let scratch_plaintext_len = key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_plaintext, &mut scratch, ) @@ -623,7 +623,7 @@ fn assert_rsaes_pkcs1v15_wycheproof_vectors( scratch_valid_checked = true; } } - "invalid" => { + WycheproofResult::Invalid => { invalid = invalid.strict_add(1); assert!( decrypted.is_err(), @@ -642,8 +642,7 @@ fn assert_rsaes_pkcs1v15_wycheproof_vectors( key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_plaintext, &mut scratch, ) @@ -660,10 +659,9 @@ fn assert_rsaes_pkcs1v15_wycheproof_vectors( scratch_invalid_checked = true; } } - "acceptable" => { + WycheproofResult::Acceptable => { acceptable = acceptable.strict_add(1); } - other => panic!("unknown Wycheproof result `{other}`"), } } if group_has_valid { @@ -716,8 +714,8 @@ fn assert_pkcs1v15_sig_gen_wycheproof_vectors(json: &str, expected_key_size: u64 let blinding_factor_inverse = fixed_width_one(key.public_key().modulus().len()); for test in test_cases(group) { - match field(test, "result") { - "valid" => { + match wycheproof_result(field(test, "result")) { + WycheproofResult::Valid => { valid = valid.strict_add(1); let message = hex_to_vec(field(test, "msg")); let expected_signature = hex_to_vec(field(test, "sig")); @@ -728,8 +726,7 @@ fn assert_pkcs1v15_sig_gen_wycheproof_vectors(json: &str, expected_key_size: u64 .sign_pkcs1v15_with_blinding_factor( profile, &message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, ) .expect("Wycheproof PKCS1v1.5 private signing must succeed"); @@ -742,8 +739,7 @@ fn assert_pkcs1v15_sig_gen_wycheproof_vectors(json: &str, expected_key_size: u64 .sign_pkcs1v15_with_blinding_factor_and_scratch( profile, &message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_signature, &mut scratch, ) @@ -758,10 +754,9 @@ fn assert_pkcs1v15_sig_gen_wycheproof_vectors(json: &str, expected_key_size: u64 .verify_pkcs1v15(profile, &message, &signature) .expect("Wycheproof PKCS1v1.5 generated signature must verify"); } - "acceptable" | "invalid" => { + WycheproofResult::Acceptable | WycheproofResult::Invalid => { skipped = skipped.strict_add(1); } - other => panic!("unknown Wycheproof result `{other}`"), } } } @@ -769,7 +764,12 @@ fn assert_pkcs1v15_sig_gen_wycheproof_vectors(json: &str, expected_key_size: u64 assert_eq!(valid, expected_valid); assert_eq!( valid.strict_add(skipped), - suite["numberOfTests"].as_u64().unwrap() as usize + usize::try_from( + suite["numberOfTests"] + .as_u64() + .expect("Wycheproof numberOfTests must be numeric") + ) + .expect("Wycheproof numberOfTests must fit usize") ); } diff --git a/tests/scrypt_differential.rs b/tests/scrypt_differential.rs index f02a4001..ca5792db 100644 --- a/tests/scrypt_differential.rs +++ b/tests/scrypt_differential.rs @@ -11,16 +11,16 @@ use proptest::{prelude::*, test_runner::Config as ProptestConfig}; use rscrypto::{Scrypt, ScryptParams}; fn oracle_scrypt(password: &[u8], salt: &[u8], log_n: u8, r: u32, p: u32, out_len: usize) -> Vec { - let params = scrypt::Params::new(log_n, r, p).unwrap(); + let params = scrypt::Params::new(log_n, r, p).expect("RustCrypto must accept generated scrypt parameters"); let mut out = vec![0u8; out_len]; - scrypt::scrypt(password, salt, ¶ms, &mut out).unwrap(); + scrypt::scrypt(password, salt, ¶ms, &mut out).expect("RustCrypto scrypt derivation must succeed"); out } fn rs_hash(password: &[u8], salt: &[u8], log_n: u8, r: u32, p: u32, out_len: usize) -> Vec { - let params = ScryptParams::new(log_n, r, p).unwrap(); + let params = ScryptParams::new(log_n, r, p).expect("generated rscrypto scrypt parameters must be valid"); let mut out = vec![0u8; out_len]; - Scrypt::derive(¶ms, password, salt, &mut out).unwrap(); + Scrypt::derive(¶ms, password, salt, &mut out).expect("rscrypto scrypt derivation must succeed"); out } @@ -54,10 +54,12 @@ proptest! { p in 1u32..=2, ) { let params = ScryptParams::new(log_n, r, p) - .unwrap(); + .expect("generated scrypt verification parameters must be valid"); let mut hash = [0u8; 32]; - Scrypt::derive(¶ms, &password, &salt, &mut hash).unwrap(); - prop_assert!(Scrypt::verify(¶ms, &password, &salt, &hash).is_ok()); + Scrypt::derive(¶ms, &password, &salt, &mut hash) + .expect("scrypt verification fixture must derive"); + Scrypt::verify(¶ms, &password, &salt, &hash) + .expect("fresh scrypt hash must verify"); } } @@ -83,29 +85,25 @@ fn scrypt_short_dklen_is_prefix_of_wide() { // `Scrypt::derive(.., out_len=1)` must equal the first byte of // `Scrypt::derive(.., out_len=64)` for the same inputs. Exercises the // `out_len=1` PBKDF2 single-byte tail that the oracle rejects. - let params_short = rscrypto::ScryptParams::new(6, 2, 1).unwrap(); - let params_wide = rscrypto::ScryptParams::new(6, 2, 1).unwrap(); + let params = ScryptParams::new(6, 2, 1).expect("short-output scrypt parameters must be valid"); let mut short_out = [0u8; 1]; let mut wide_out = [0u8; 64]; - Scrypt::derive(¶ms_short, b"pw", b"salty-salty-salt", &mut short_out).unwrap(); - Scrypt::derive(¶ms_wide, b"pw", b"salty-salty-salt", &mut wide_out).unwrap(); + Scrypt::derive(¶ms, b"pw", b"salty-salty-salt", &mut short_out).expect("one-byte scrypt derivation must succeed"); + Scrypt::derive(¶ms, b"pw", b"salty-salty-salt", &mut wide_out).expect("64-byte scrypt derivation must succeed"); assert_eq!(short_out[0], wide_out[0]); } #[test] fn scrypt_verify_rejects_byte_flip_at_every_position() { - let params = ScryptParams::new(6, 2, 1).unwrap(); + let params = ScryptParams::new(6, 2, 1).expect("byte-flip scrypt parameters must be valid"); let password = b"correct horse battery staple"; let salt = b"random-salt-1234"; let mut hash = [0u8; 32]; - Scrypt::derive(¶ms, password, salt, &mut hash).unwrap(); + Scrypt::derive(¶ms, password, salt, &mut hash).expect("byte-flip scrypt fixture must derive"); for pos in 0..hash.len() { let mut tampered = hash; tampered[pos] ^= 0x01; - assert!( - Scrypt::verify(¶ms, password, salt, &tampered).is_err(), - "verify must reject flip at byte {pos}", - ); + Scrypt::verify(¶ms, password, salt, &tampered).expect_err("scrypt must reject a hash with any flipped byte"); } } diff --git a/tests/secret_redaction.rs b/tests/secret_redaction.rs index 227766eb..96c1df68 100644 --- a/tests/secret_redaction.rs +++ b/tests/secret_redaction.rs @@ -11,7 +11,21 @@ fn generic_secret_owner_debug_is_redacted() { #[test] fn keyed_state_debug_snapshots_are_redacted() { + #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv", feature = "ascon-aead"))] const KEY_16: [u8; 16] = [0x53; 16]; + #[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "hkdf", + feature = "hmac", + feature = "hmac-sha3", + feature = "kmac", + feature = "pbkdf2", + feature = "poly1305", + feature = "xchacha20poly1305" + ))] const KEY_32: [u8; 32] = [0x53; 32]; #[cfg(feature = "aes-gcm")] @@ -132,11 +146,13 @@ fn keyed_state_debug_snapshots_are_redacted() { #[test] fn keyed_hash_debug_snapshots_are_redacted() { + #[cfg(any(feature = "blake2b", feature = "blake2s", feature = "blake3"))] const KEY: [u8; 32] = [0x53; 32]; #[cfg(feature = "blake2s")] { - let params = rscrypto::Blake2sParams::new().key(rscrypto::Blake2sKey::new(&KEY).unwrap()); + let key = rscrypto::Blake2sKey::new(&KEY).expect("32-byte BLAKE2s key must be valid"); + let params = rscrypto::Blake2sParams::new().key(key); assert_debug_snapshot( ¶ms, "Blake2sParams { key_len: 32, salt: [0, 0, 0, 0, 0, 0, 0, 0], personal: [0, 0, 0, 0, 0, 0, 0, 0] }", @@ -147,7 +163,8 @@ fn keyed_hash_debug_snapshots_are_redacted() { #[cfg(feature = "blake2b")] { - let params = rscrypto::Blake2bParams::new().key(rscrypto::Blake2bKey::new(&KEY).unwrap()); + let key = rscrypto::Blake2bKey::new(&KEY).expect("32-byte BLAKE2b key must be valid"); + let params = rscrypto::Blake2bParams::new().key(key); assert_debug_snapshot( ¶ms, "Blake2bParams { key_len: 32, salt: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], personal: [0, 0, 0, 0, 0, \ @@ -155,7 +172,8 @@ fn keyed_hash_debug_snapshots_are_redacted() { ); assert_debug_snapshot(¶ms.build_256(), "Blake2b256 { .. }"); assert_debug_snapshot(¶ms.build_512(), "Blake2b512 { .. }"); - assert_debug_snapshot(¶ms.build(32).unwrap(), "Blake2b { output_len: 32, .. }"); + let variable = params.build(32).expect("32-byte BLAKE2b output must be valid"); + assert_debug_snapshot(&variable, "Blake2b { output_len: 32, .. }"); } #[cfg(feature = "blake3")] @@ -243,7 +261,8 @@ fn private_key_and_shared_secret_debug_snapshots_are_redacted() { let (_, key) = <$profile>::generate_keypair(|out| { for (index, byte) in out.iter_mut().enumerate() { - *byte = 0x53u8.wrapping_add(index as u8); + let index = u8::try_from(index).expect("ML-KEM entropy index must fit in u8"); + *byte = 0x53u8.wrapping_add(index); } Ok::<(), rscrypto::MlKemError>(()) }) @@ -297,7 +316,8 @@ fn secret_input_error_snapshots_do_not_echo_input_bytes() { impl core::error::Error for EntropyStateError {} - let passwords = rscrypto::Argon2idPassword::new(rscrypto::Argon2Params::new(32, 2, 1).unwrap()).unwrap(); + let params = rscrypto::Argon2Params::new(32, 2, 1).expect("test Argon2 parameters must be valid"); + let passwords = rscrypto::Argon2idPassword::new(params).expect("test Argon2 policy must be valid"); let error = passwords .hash_password_with(b"password", |_| Err(EntropyStateError)) .expect_err("entropy-source failure must be preserved as an opaque error"); diff --git a/tests/serde_roundtrip.rs b/tests/serde_roundtrip.rs index aedb01b4..ecbe9c7c 100644 --- a/tests/serde_roundtrip.rs +++ b/tests/serde_roundtrip.rs @@ -13,7 +13,8 @@ macro_rules! serde_roundtrip { let bytes = { let mut b = [0u8; $len]; for (i, v) in b.iter_mut().enumerate() { - *v = (i as u8).wrapping_mul(0x37).wrapping_add(0x11); + let i = u8::try_from(i).expect("serde fixture index must fit in one byte"); + *v = i.wrapping_mul(0x37).wrapping_add(0x11); } b }; @@ -154,10 +155,10 @@ fn wrong_length_bytes_rejected() { use rscrypto::aead::Nonce96; // Nonce96 is 12 bytes; feeding 11 or 13 must fail. - let short = serde_json::to_string(&[0u8; 11]).unwrap(); - let long = serde_json::to_string(&[0u8; 13]).unwrap(); - assert!(serde_json::from_str::(&short).is_err()); - assert!(serde_json::from_str::(&long).is_err()); + let short = serde_json::to_string(&[0u8; 11]).expect("short nonce fixture must serialize"); + let long = serde_json::to_string(&[0u8; 13]).expect("long nonce fixture must serialize"); + serde_json::from_str::(&short).expect_err("11-byte nonce must be rejected"); + serde_json::from_str::(&long).expect_err("13-byte nonce must be rejected"); } #[cfg(all(feature = "serde-secrets", feature = "aes-gcm"))] @@ -165,11 +166,12 @@ fn wrong_length_bytes_rejected() { fn secret_sequence_rejects_partial_and_wrong_length_inputs() { use rscrypto::Aes256GcmKey; - assert!(serde_json::from_str::("[17,34]").is_err()); - assert!(serde_json::from_str::("[17,\"not-a-byte\"]").is_err()); + serde_json::from_str::("[17,34]").expect_err("partial AES-256-GCM key must be rejected"); + serde_json::from_str::("[17,\"not-a-byte\"]") + .expect_err("non-byte AES-256-GCM key element must be rejected"); let exact = vec![0x42u8; Aes256GcmKey::LENGTH]; - let encoded = serde_json::to_string(&exact).unwrap(); - let key = serde_json::from_str::(&encoded).unwrap(); + let encoded = serde_json::to_string(&exact).expect("exact-length key fixture must serialize"); + let key = serde_json::from_str::(&encoded).expect("exact-length AES-256-GCM key must deserialize"); assert_eq!(key.as_bytes(), exact.as_slice()); } diff --git a/tests/sha256_differential.rs b/tests/sha256_differential.rs index 5ab6860c..27bc3e3e 100644 --- a/tests/sha256_differential.rs +++ b/tests/sha256_differential.rs @@ -35,8 +35,8 @@ proptest! { let mut h = Sha256::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -56,8 +56,8 @@ proptest! { let mut h = Sha224::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } diff --git a/tests/sha256_official_vectors.rs b/tests/sha256_official_vectors.rs index a72bcb6f..4cce9e21 100644 --- a/tests/sha256_official_vectors.rs +++ b/tests/sha256_official_vectors.rs @@ -3,16 +3,16 @@ mod support; use rscrypto::hashes::crypto::Sha256; -use support::blobby_compat::Blob2Iterator; +use support::blobby_compat::BlobIterator; #[test] fn sha256_official_vectors() { let data = include_bytes!("../testdata/sha2/sha256.blb"); - for (i, row) in Blob2Iterator::new(data) + for (i, row) in BlobIterator::<2>::new(data) .expect("sha256 vector corpus must parse") .enumerate() { - let [input, output] = row.unwrap_or_else(|err| panic!("sha256 vector row decode failed at case {i}: {err:?}")); + let [input, output] = row.expect("SHA-256 vector row must decode"); let actual = Sha256::digest(input); assert_eq!( &actual[..], diff --git a/tests/sha2_official_vectors.rs b/tests/sha2_official_vectors.rs index 8310d429..03ff0970 100644 --- a/tests/sha2_official_vectors.rs +++ b/tests/sha2_official_vectors.rs @@ -3,14 +3,14 @@ mod support; use rscrypto::hashes::crypto::{Sha224, Sha384, Sha512, Sha512_256}; -use support::blobby_compat::Blob2Iterator; +use support::blobby_compat::BlobIterator; fn run_fixed_vectors(data: &'static [u8], name: &str, mut digest: impl FnMut(&[u8]) -> [u8; OUT]) { - for (i, row) in Blob2Iterator::new(data) + for (i, row) in BlobIterator::<2>::new(data) .expect("sha2 vector corpus must parse") .enumerate() { - let [input, output] = row.unwrap_or_else(|err| panic!("{name} vector row decode failed at case {i}: {err:?}")); + let [input, output] = row.expect("SHA-2 vector row must decode"); let actual = digest(input); assert_eq!( &actual[..], diff --git a/tests/sha3_differential.rs b/tests/sha3_differential.rs index 28873415..dd6175f2 100644 --- a/tests/sha3_differential.rs +++ b/tests/sha3_differential.rs @@ -66,8 +66,8 @@ proptest! { let mut h = Sha3_224::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -82,8 +82,8 @@ proptest! { let mut h = Sha3_256::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -98,8 +98,8 @@ proptest! { let mut h = Sha3_384::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -114,8 +114,8 @@ proptest! { let mut h = Sha3_512::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } diff --git a/tests/sha3_official_vectors.rs b/tests/sha3_official_vectors.rs index 298d067b..e6b2263c 100644 --- a/tests/sha3_official_vectors.rs +++ b/tests/sha3_official_vectors.rs @@ -7,14 +7,14 @@ use rscrypto::{ hashes::crypto::{Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256}, traits::Xof as _, }; -use support::blobby_compat::Blob2Iterator; +use support::blobby_compat::BlobIterator; fn run_fixed_vectors(data: &'static [u8], name: &str, mut digest: impl FnMut(&[u8]) -> [u8; OUT]) { - for (i, row) in Blob2Iterator::new(data) + for (i, row) in BlobIterator::<2>::new(data) .expect("sha3 vector corpus must parse") .enumerate() { - let [input, output] = row.unwrap_or_else(|err| panic!("{name} vector row decode failed at case {i}: {err:?}")); + let [input, output] = row.expect("SHA-3 vector row must decode"); let actual = digest(input); assert_eq!( &actual[..], @@ -50,11 +50,11 @@ fn sha3_384_official_vectors() { } fn run_xof_vectors(data: &'static [u8], name: &str, mut xof: impl FnMut(&[u8], &mut [u8])) { - for (i, row) in Blob2Iterator::new(data) + for (i, row) in BlobIterator::<2>::new(data) .expect("sha3 xof vector corpus must parse") .enumerate() { - let [input, output] = row.unwrap_or_else(|err| panic!("{name} vector row decode failed at case {i}: {err:?}")); + let [input, output] = row.expect("SHA-3 XOF vector row must decode"); let mut out = vec![0u8; output.len()]; xof(input, &mut out); assert_eq!( diff --git a/tests/sha512_differential.rs b/tests/sha512_differential.rs index 2c4e1857..04f425cc 100644 --- a/tests/sha512_differential.rs +++ b/tests/sha512_differential.rs @@ -43,8 +43,8 @@ proptest! { let mut h = Sha512::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -64,8 +64,8 @@ proptest! { let mut h = Sha384::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -85,8 +85,8 @@ proptest! { let mut h = Sha512_256::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } diff --git a/tests/shake128_differential.rs b/tests/shake128_differential.rs index f30075b7..153dc2cf 100644 --- a/tests/shake128_differential.rs +++ b/tests/shake128_differential.rs @@ -36,8 +36,8 @@ proptest! { let mut h = Shake128::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = usize::from(data[i]).strict_rem(97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -55,7 +55,7 @@ proptest! { out_len in 0usize..2048, split in any::(), ) { - let split = split % (out_len + 1); + let split = split.strict_rem(out_len.strict_add(1)); let mut expected = vec![0u8; out_len]; { diff --git a/tests/shake256_differential.rs b/tests/shake256_differential.rs index 64a558da..22bfd94e 100644 --- a/tests/shake256_differential.rs +++ b/tests/shake256_differential.rs @@ -36,8 +36,8 @@ proptest! { let mut h = Shake256::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = usize::from(data[i]).strict_rem(97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -55,7 +55,7 @@ proptest! { out_len in 0usize..2048, split in any::(), ) { - let split = split % (out_len + 1); + let split = split.strict_rem(out_len.strict_add(1)); let mut expected = vec![0u8; out_len]; { diff --git a/tests/support/blobby_compat.rs b/tests/support/blobby_compat.rs index 3424fa0e..fed8cfcf 100644 --- a/tests/support/blobby_compat.rs +++ b/tests/support/blobby_compat.rs @@ -1,7 +1,5 @@ -#![allow(dead_code)] - #[derive(Debug, Eq, PartialEq, Copy, Clone)] -pub enum Error { +pub(crate) enum Error { InvalidVlq, InvalidIndex, UnexpectedEnd, @@ -13,9 +11,9 @@ const VAL_MASK: u8 = 0b0111_1111; fn read_vlq(data: &[u8], pos: &mut usize) -> Result { let b = *data.get(*pos).ok_or(Error::UnexpectedEnd)?; - *pos += 1; + *pos = (*pos).strict_add(1); let mut next = b & NEXT_MASK; - let mut val = (b & VAL_MASK) as usize; + let mut val = usize::from(b & VAL_MASK); macro_rules! step { () => { @@ -23,10 +21,10 @@ fn read_vlq(data: &[u8], pos: &mut usize) -> Result { return Ok(val); } let b = *data.get(*pos).ok_or(Error::UnexpectedEnd)?; - *pos += 1; + *pos = (*pos).strict_add(1); next = b & NEXT_MASK; - let t = (b & VAL_MASK) as usize; - val = ((val + 1) << 7) + t; + let t = usize::from(b & VAL_MASK); + val = val.strict_add(1).strict_mul(128).strict_add(t); }; } @@ -41,14 +39,18 @@ fn read_vlq(data: &[u8], pos: &mut usize) -> Result { Ok(val) } -pub struct BlobIterator<'a> { +pub(crate) struct BlobIterator<'a, const N: usize> { data: &'a [u8], dedup: Box<[&'a [u8]]>, pos: usize, } -impl<'a> BlobIterator<'a> { - pub fn new(data: &'a [u8]) -> Result { +impl<'a, const N: usize> BlobIterator<'a, N> { + pub(crate) fn new(data: &'a [u8]) -> Result { + if N == 0 { + return Err(Error::NotEnoughElements); + } + let mut pos = 0; let dedup_n = read_vlq(data, &mut pos)?; @@ -61,7 +63,7 @@ impl<'a> BlobIterator<'a> { } Ok(Self { - data: &data[pos..], + data: data.get(pos..).ok_or(Error::UnexpectedEnd)?, dedup: dedup.into_boxed_slice(), pos: 0, }) @@ -87,58 +89,28 @@ impl<'a> BlobIterator<'a> { } } -impl<'a> Iterator for BlobIterator<'a> { - type Item = Result<&'a [u8], Error>; +impl<'a, const N: usize> Iterator for BlobIterator<'a, N> { + type Item = Result<[&'a [u8]; N], Error>; fn next(&mut self) -> Option { if self.pos >= self.data.len() { return None; } - let value = self.read(); - if value.is_err() { - self.error_block(); - } - Some(value) - } -} - -macro_rules! blob_iter { - ($name:ident, $n:expr) => { - pub struct $name<'a> { - inner: BlobIterator<'a>, - } - - impl<'a> $name<'a> { - pub fn new(data: &'a [u8]) -> Result { - BlobIterator::new(data).map(|inner| Self { inner }) + let mut out = [&[][..]; N]; + for slot in &mut out { + if self.pos >= self.data.len() { + self.error_block(); + return Some(Err(Error::NotEnoughElements)); } - } - - impl<'a> Iterator for $name<'a> { - type Item = Result<[&'a [u8]; $n], Error>; - - fn next(&mut self) -> Option { - let mut out = [&[][..]; $n]; - - for (i, slot) in out.iter_mut().enumerate() { - *slot = match self.inner.next() { - Some(Ok(value)) => value, - Some(Err(err)) => return Some(Err(err)), - None if i == 0 => return None, - None => { - self.inner.error_block(); - return Some(Err(Error::NotEnoughElements)); - } - }; + *slot = match self.read() { + Ok(value) => value, + Err(err) => { + self.error_block(); + return Some(Err(err)); } - - Some(Ok(out)) - } + }; } - }; + Some(Ok(out)) + } } - -blob_iter!(Blob2Iterator, 2); -blob_iter!(Blob3Iterator, 3); -blob_iter!(Blob6Iterator, 6); diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 65835047..db0af257 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -3,4 +3,4 @@ //! This stays intentionally small. Right now it only exposes vector-corpus //! parsing helpers used by the official test suites. -pub mod blobby_compat; +pub(crate) mod blobby_compat; diff --git a/tests/x25519_vectors.rs b/tests/x25519_vectors.rs index 29dfa028..652a91f8 100644 --- a/tests/x25519_vectors.rs +++ b/tests/x25519_vectors.rs @@ -4,9 +4,11 @@ use rscrypto::{X25519PublicKey, X25519SecretKey, X25519SharedSecret}; use x25519_dalek::{PublicKey as DalekPublicKey, StaticSecret as DalekStaticSecret}; mod common; +#[path = "common/array.rs"] +mod hex_array; fn decode_hex_32(hex: &str) -> [u8; 32] { - common::decode_hex_array::<32>(hex) + hex_array::decode_hex_array::<32>(hex) } fn iterative_x25519(iterations: usize) -> [u8; 32] { @@ -14,7 +16,11 @@ fn iterative_x25519(iterations: usize) -> [u8; 32] { let mut u = X25519PublicKey::basepoint(); for _ in 0..iterations { - let next = k.diffie_hellman(&u).unwrap().expose_secret().expose(); + let next = k + .diffie_hellman(&u) + .expect("RFC 7748 iterative ladder must produce a nonzero secret") + .expose_secret() + .expose(); u = X25519PublicKey::from_bytes(*k.as_bytes()); k = X25519SecretKey::from_bytes(next); } @@ -32,7 +38,10 @@ fn rfc_7748_scalar_multiplication_vectors_match() { )); let expected = decode_hex_32("c3da55379de9c6908e94ea4df28d084f32eccf03491c71f754b4075577a28552"); - assert_eq!(*scalar.diffie_hellman(&public).unwrap().as_bytes(), expected); + let shared = scalar + .diffie_hellman(&public) + .expect("first RFC 7748 scalar-multiplication vector must produce a nonzero secret"); + assert_eq!(*shared.as_bytes(), expected); let scalar = X25519SecretKey::from_bytes(decode_hex_32( "4b66e9d4d1b4673c5ad22691957d6af5c11b6421e0ea01d42ca4169e7918ba0d", @@ -42,7 +51,10 @@ fn rfc_7748_scalar_multiplication_vectors_match() { )); let expected = decode_hex_32("95cbde9476e8907d7aade45cb4b873f88b595a68799fa152e6f8f7647aac7957"); - assert_eq!(*scalar.diffie_hellman(&public).unwrap().as_bytes(), expected); + let shared = scalar + .diffie_hellman(&public) + .expect("second RFC 7748 scalar-multiplication vector must produce a nonzero secret"); + assert_eq!(*shared.as_bytes(), expected); } #[test] @@ -73,8 +85,12 @@ fn rfc_7748_diffie_hellman_vector_matches() { )); let expected = decode_hex_32("4a5d9d5ba4ce2de1728e3bf480350f25e07e21c947d19e3376f09b3c1e161742"); - let alice_shared = alice.diffie_hellman(&bob_public).unwrap(); - let bob_shared = bob.diffie_hellman(&alice_public).unwrap(); + let alice_shared = alice + .diffie_hellman(&bob_public) + .expect("Alice RFC 7748 exchange must produce a nonzero secret"); + let bob_shared = bob + .diffie_hellman(&alice_public) + .expect("Bob RFC 7748 exchange must produce a nonzero secret"); assert_eq!(alice.public_key(), alice_public); assert_eq!(bob.public_key(), bob_public); @@ -87,8 +103,11 @@ fn low_order_points_return_all_zero_error() { let secret = X25519SecretKey::from_bytes([0x42; X25519SecretKey::LENGTH]); let low_order = X25519PublicKey::from_bytes([0u8; X25519PublicKey::LENGTH]); - assert!(secret.diffie_hellman(&low_order).is_err()); - assert!(X25519SharedSecret::diffie_hellman(&secret, &low_order).is_err()); + secret + .diffie_hellman(&low_order) + .expect_err("X25519 must reject a low-order point"); + X25519SharedSecret::diffie_hellman(&secret, &low_order) + .expect_err("X25519 shared-secret API must reject a low-order point"); } #[test] @@ -98,10 +117,9 @@ fn non_canonical_public_inputs_are_accepted_and_reduced() { "edffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7f", )); - assert!( - secret.diffie_hellman(&public).is_err(), - "u = p should reduce to zero and fail the all-zero check" - ); + secret + .diffie_hellman(&public) + .expect_err("u = p must reduce to zero and fail the all-zero check"); assert_eq!( public.to_bytes(), decode_hex_32("edffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7f") @@ -113,7 +131,8 @@ fn public_keys_and_shared_secrets_match_x25519_dalek() { for seed in 0u8..32 { let mut secret_bytes = [0u8; 32]; for (index, byte) in secret_bytes.iter_mut().enumerate() { - *byte = seed.wrapping_mul(17).wrapping_add((index as u8).wrapping_mul(29)); + let index = u8::try_from(index).expect("32-byte key index must fit in one byte"); + *byte = seed.wrapping_mul(17).wrapping_add(index.wrapping_mul(29)); } let ours_secret = X25519SecretKey::from_bytes(secret_bytes); @@ -125,9 +144,10 @@ fn public_keys_and_shared_secrets_match_x25519_dalek() { let mut peer_bytes = [0u8; 32]; for (index, byte) in peer_bytes.iter_mut().enumerate() { + let index = u8::try_from(index).expect("32-byte peer-key index must fit in one byte"); *byte = seed .wrapping_mul(23) - .wrapping_add((index as u8).wrapping_mul(11)) + .wrapping_add(index.wrapping_mul(11)) .wrapping_add(5); } peer_bytes[31] |= 0x80; @@ -138,9 +158,10 @@ fn public_keys_and_shared_secrets_match_x25519_dalek() { let dalek_shared = dalek_secret.diffie_hellman(&dalek_peer).to_bytes(); if dalek_shared.iter().all(|&byte| byte == 0) { - assert!(ours_shared.is_err()); + ours_shared.expect_err("rscrypto must reject every all-zero X25519 shared secret"); } else { - assert_eq!(*ours_shared.unwrap().as_bytes(), dalek_shared); + let ours_shared = ours_shared.expect("nonzero x25519-dalek shared secret must be accepted"); + assert_eq!(*ours_shared.as_bytes(), dalek_shared); } } } @@ -150,12 +171,15 @@ fn public_key_matches_basepoint_diffie_hellman() { for seed in 0u8..32 { let mut secret_bytes = [0u8; 32]; for (index, byte) in secret_bytes.iter_mut().enumerate() { - *byte = seed.wrapping_mul(41).wrapping_add((index as u8).wrapping_mul(7)); + let index = u8::try_from(index).expect("32-byte key index must fit in one byte"); + *byte = seed.wrapping_mul(41).wrapping_add(index.wrapping_mul(7)); } let secret = X25519SecretKey::from_bytes(secret_bytes); let public = secret.public_key(); - let via_ladder = secret.diffie_hellman(&X25519PublicKey::basepoint()).unwrap(); + let via_ladder = secret + .diffie_hellman(&X25519PublicKey::basepoint()) + .expect("basepoint exchange must produce a nonzero X25519 secret"); assert_eq!(public.to_bytes(), *via_ladder.as_bytes()); } @@ -165,7 +189,9 @@ fn public_key_matches_basepoint_diffie_hellman() { fn secret_and_shared_debug_are_masked() { let secret = X25519SecretKey::from_bytes([0x42; 32]); let public = secret.public_key(); - let shared = secret.diffie_hellman(&public).unwrap(); + let shared = secret + .diffie_hellman(&public) + .expect("self-derived public key must produce a nonzero X25519 secret"); assert_eq!(format!("{secret:?}"), "X25519SecretKey(****)"); assert_eq!(format!("{shared:?}"), "X25519SharedSecret(****)"); diff --git a/tests/x25519_wycheproof.rs b/tests/x25519_wycheproof.rs index 38dde501..43012b98 100644 --- a/tests/x25519_wycheproof.rs +++ b/tests/x25519_wycheproof.rs @@ -4,7 +4,9 @@ use rscrypto::{X25519PublicKey, X25519SecretKey}; use serde_json::Value; mod common; -use common::decode_hex_array; +#[path = "common/array.rs"] +mod hex_array; +use hex_array::decode_hex_array; const X25519: &str = include_str!("../testdata/auth/wycheproof/x25519_test.json"); @@ -16,9 +18,11 @@ struct Counts { } fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] + value + .get(name) + .expect("Wycheproof field must be present") .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + .expect("Wycheproof field must be a string") } fn groups(suite: &Value) -> &[Value] { @@ -52,26 +56,22 @@ fn wycheproof_x25519_vectors_match_or_reject_all_zero_shared_secret() { let expected_shared: [u8; 32] = decode_hex_array(field(test, "shared")); let result = secret.diffie_hellman(&public); - match field(test, "result") { - "valid" => counts.valid = counts.valid.strict_add(1), - "acceptable" => counts.acceptable = counts.acceptable.strict_add(1), - other => panic!("unsupported Wycheproof X25519 result `{other}`"), + let disposition = field(test, "result"); + assert!( + matches!(disposition, "valid" | "acceptable"), + "unsupported Wycheproof X25519 result `{disposition}`" + ); + if disposition == "valid" { + counts.valid = counts.valid.strict_add(1); + } else { + counts.acceptable = counts.acceptable.strict_add(1); } if expected_shared == [0u8; 32] { counts.zero_shared_rejected = counts.zero_shared_rejected.strict_add(1); - assert!( - result.is_err(), - "Wycheproof X25519 tcId {} must reject all-zero shared secret", - test["tcId"] - ); + result.expect_err("Wycheproof X25519 all-zero shared secret must be rejected"); } else { - let shared = result.unwrap_or_else(|error| { - panic!( - "Wycheproof X25519 tcId {} unexpectedly rejected non-zero shared secret: {error}", - test["tcId"] - ) - }); + let shared = result.expect("Wycheproof X25519 nonzero shared secret must be accepted"); assert_eq!( *shared.as_bytes(), expected_shared, diff --git a/tests/xchacha20poly1305.rs b/tests/xchacha20poly1305.rs index 816d16f7..355ce3e5 100644 --- a/tests/xchacha20poly1305.rs +++ b/tests/xchacha20poly1305.rs @@ -15,9 +15,10 @@ use common::decode_hex_vec as decode_hex; fn pattern_bytes(len: usize, seed: u8) -> Vec { let mut out = vec![0u8; len]; for (index, byte) in out.iter_mut().enumerate() { + let index = index.to_le_bytes()[0]; *byte = seed - .wrapping_add((index as u8).wrapping_mul(19)) - .wrapping_add((index as u8).rotate_left(2)); + .wrapping_add(index.wrapping_mul(19)) + .wrapping_add(index.rotate_left(2)); } out } @@ -35,18 +36,26 @@ fn xchacha20poly1305_matches_draft_vector() { ); let expected_tag = decode_hex("c0875924c1c7987947deafd8780acf49"); - let key = XChaCha20Poly1305Key::from_bytes(key.try_into().unwrap()); - let nonce = Nonce192::from_bytes(nonce.try_into().unwrap()); + let key = XChaCha20Poly1305Key::from_bytes(key.try_into().expect("draft XChaCha20-Poly1305 key must be 32 bytes")); + let nonce = Nonce192::from_bytes( + nonce + .try_into() + .expect("draft XChaCha20-Poly1305 nonce must be 24 bytes"), + ); let cipher = XChaCha20Poly1305::new(&key); let mut sealed = vec![0u8; plaintext.len() + XChaCha20Poly1305::TAG_SIZE]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut sealed).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut sealed) + .expect("draft XChaCha20-Poly1305 seal buffer must fit plaintext and tag"); assert_eq!(&sealed[..plaintext.len()], expected_ciphertext.as_slice()); assert_eq!(&sealed[plaintext.len()..], expected_tag.as_slice()); let mut opened = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &aad, &sealed, &mut opened).unwrap(); + cipher + .decrypt(&nonce, &aad, &sealed, &mut opened) + .expect("draft XChaCha20-Poly1305 ciphertext and tag must authenticate"); assert_eq!(opened, plaintext); } @@ -65,18 +74,22 @@ fn xchacha20poly1305_matches_rustcrypto_oracle() { let oracle_nonce = Array(nonce_bytes); let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto XChaCha20-Poly1305 oracle input must seal"); let mut oracle_buffer = plaintext.to_vec(); let oracle_tag = oracle .encrypt_inout_detached(&oracle_nonce, aad, oracle_buffer.as_mut_slice().into()) - .unwrap(); + .expect("RustCrypto XChaCha20-Poly1305 oracle input must seal"); assert_eq!(ours, oracle_buffer); assert_eq!(tag.as_bytes(), oracle_tag.as_slice()); let typed_tag = XChaCha20Poly1305Tag::from_bytes(tag.to_bytes()); - cipher.decrypt_in_place(&nonce, aad, &mut ours, &typed_tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &typed_tag) + .expect("fresh rscrypto XChaCha20-Poly1305 ciphertext must authenticate"); assert_eq!(ours, plaintext); } @@ -87,22 +100,23 @@ fn xchacha20poly1305_rejects_modified_tag() { let cipher = XChaCha20Poly1305::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("XChaCha20-Poly1305 forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &XChaCha20Poly1305Tag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &XChaCha20Poly1305Tag::from_bytes(tag)) + .expect_err("modified XChaCha20-Poly1305 tag must fail authentication"); } #[test] fn xchacha20poly1305_rejects_wrong_tag_length() { - assert!(XChaCha20Poly1305::tag_from_slice(&[0u8; 0]).is_err()); - assert!(XChaCha20Poly1305::tag_from_slice(&[0u8; 15]).is_err()); - assert!(XChaCha20Poly1305::tag_from_slice(&[0u8; 17]).is_err()); - assert!(XChaCha20Poly1305::tag_from_slice(&[0u8; 16]).is_ok()); + XChaCha20Poly1305::tag_from_slice(&[0u8; 0]).expect_err("empty XChaCha20-Poly1305 tag must be rejected"); + XChaCha20Poly1305::tag_from_slice(&[0u8; 15]).expect_err("short XChaCha20-Poly1305 tag must be rejected"); + XChaCha20Poly1305::tag_from_slice(&[0u8; 17]).expect_err("long XChaCha20-Poly1305 tag must be rejected"); + let _tag = XChaCha20Poly1305::tag_from_slice(&[0u8; 16]).expect("16-byte XChaCha20-Poly1305 tag must be accepted"); } #[test] @@ -128,7 +142,9 @@ fn xchacha20poly1305_boundary_and_large_inputs_match_oracle() { let aad = pattern_bytes(aad_len, 0xc4); let mut combined = vec![0u8; plaintext_len + XChaCha20Poly1305::TAG_SIZE]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut combined).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut combined) + .expect("rscrypto combined XChaCha20-Poly1305 oracle input must seal"); let oracle_combined = oracle .encrypt( @@ -138,14 +154,16 @@ fn xchacha20poly1305_boundary_and_large_inputs_match_oracle() { aad: &aad, }, ) - .unwrap(); + .expect("RustCrypto combined XChaCha20-Poly1305 oracle input must seal"); assert_eq!( combined, oracle_combined, "combined ciphertext mismatch pt_len={plaintext_len} aad_len={aad_len}" ); let mut opened = vec![0u8; plaintext_len]; - cipher.decrypt(&nonce, &aad, &oracle_combined, &mut opened).unwrap(); + cipher + .decrypt(&nonce, &aad, &oracle_combined, &mut opened) + .expect("RustCrypto XChaCha20-Poly1305 ciphertext must open in rscrypto"); assert_eq!( opened, plaintext, "combined decrypt mismatch pt_len={plaintext_len} aad_len={aad_len}" @@ -159,14 +177,16 @@ fn xchacha20poly1305_boundary_and_large_inputs_match_oracle() { aad: &aad, }, ) - .unwrap(); + .expect("rscrypto XChaCha20-Poly1305 ciphertext must open in RustCrypto"); assert_eq!( oracle_opened, plaintext, "oracle decrypt mismatch pt_len={plaintext_len} aad_len={aad_len}" ); let mut detached = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut detached).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut detached) + .expect("rscrypto detached XChaCha20-Poly1305 oracle input must seal"); assert_eq!( detached, oracle_combined[..plaintext_len], @@ -181,7 +201,7 @@ fn xchacha20poly1305_boundary_and_large_inputs_match_oracle() { let typed_tag = XChaCha20Poly1305Tag::from_bytes(tag.to_bytes()); cipher .decrypt_in_place(&nonce, &aad, &mut detached, &typed_tag) - .unwrap(); + .expect("fresh detached XChaCha20-Poly1305 ciphertext must authenticate"); assert_eq!( detached, plaintext, "detached decrypt mismatch pt_len={plaintext_len} aad_len={aad_len}" diff --git a/tools/ct-binsec-harness/src/main.rs b/tools/ct-binsec-harness/src/main.rs index d000e60f..3499d8a5 100644 --- a/tools/ct-binsec-harness/src/main.rs +++ b/tools/ct-binsec-harness/src/main.rs @@ -2,10 +2,9 @@ use core::ptr; use rscrypto::aead::expert::AeadWithNonce; use rscrypto::{ - Aegis256, Aegis256Key, Aes128Gcm, Aes128GcmKey, Aes128GcmSiv, Aes128GcmSivKey, Aes256Gcm, Aes256GcmKey, - Aes256GcmSiv, Aes256GcmSivKey, AsconAead128, AsconAead128Key, Blake3KeyedHash, ChaCha20Poly1305, - ChaCha20Poly1305Key, HmacSha384Tag, HmacSha512Tag, Kmac256, X25519SecretKey, XChaCha20Poly1305, - XChaCha20Poly1305Key, + Aegis256, Aegis256Key, Aes128Gcm, Aes128GcmKey, Aes128GcmSiv, Aes128GcmSivKey, Aes256Gcm, Aes256GcmKey, Aes256GcmSiv, + Aes256GcmSivKey, AsconAead128, AsconAead128Key, Blake3KeyedHash, ChaCha20Poly1305, ChaCha20Poly1305Key, + HmacSha384Tag, HmacSha512Tag, Kmac256, X25519SecretKey, XChaCha20Poly1305, XChaCha20Poly1305Key, aead::{Nonce96, Nonce128, Nonce192, Nonce256}, }; @@ -20,20 +19,31 @@ const RSA_WINDOW_TABLE_LIMBS: usize = 16 * RSA_WINDOW_LIMBS; #[unsafe(no_mangle)] #[inline(never)] +/// C-compatible volatile byte fill for the analysis harness. +/// +/// # Safety +/// +/// `dst` must be valid and writable for `len` bytes. pub unsafe extern "C" fn memset(dst: *mut u8, value: i32, len: usize) -> *mut u8 { let mut offset = 0usize; while offset < len { // SAFETY: C callers require `dst..dst+len` to be writable. Volatile writes // keep zeroization visible to BINSEC and prevent this shim from becoming a // recursive compiler intrinsic. - unsafe { ptr::write_volatile(dst.add(offset), value as u8) }; - offset += 1; + unsafe { ptr::write_volatile(dst.add(offset), value.to_le_bytes()[0]) }; + offset = offset.strict_add(1); } dst } #[unsafe(no_mangle)] #[inline(never)] +/// C-compatible overlapping byte copy for the analysis harness. +/// +/// # Safety +/// +/// `src` must be readable and `dst` writable for `len` bytes. Both pointers +/// must satisfy Rust's pointer provenance requirements for those ranges. pub unsafe extern "C" fn memmove(dst: *mut u8, src: *const u8, len: usize) -> *mut u8 { if (dst as usize) <= (src as usize) { let mut offset = 0usize; @@ -42,12 +52,12 @@ pub unsafe extern "C" fn memmove(dst: *mut u8, src: *const u8, len: usize) -> *m let byte = unsafe { ptr::read(src.add(offset)) }; // SAFETY: C callers require source and destination ranges to be valid. unsafe { ptr::write(dst.add(offset), byte) }; - offset += 1; + offset = offset.strict_add(1); } } else { let mut remaining = len; while remaining != 0 { - remaining -= 1; + remaining = remaining.strict_sub(1); // SAFETY: C callers require source and destination ranges to be valid. let byte = unsafe { ptr::read(src.add(remaining)) }; // SAFETY: C callers require source and destination ranges to be valid. @@ -59,6 +69,12 @@ pub unsafe extern "C" fn memmove(dst: *mut u8, src: *const u8, len: usize) -> *m #[unsafe(no_mangle)] #[inline(never)] +/// C-compatible non-overlapping byte copy for the analysis harness. +/// +/// # Safety +/// +/// `src` must be readable and `dst` writable for `len` bytes, and the two +/// ranges must not overlap. pub unsafe extern "C" fn memcpy(dst: *mut u8, src: *const u8, len: usize) -> *mut u8 { // SAFETY: `memcpy` has the same validity contract as `memmove`, with the // additional non-overlap precondition. The `memmove` implementation is valid @@ -68,6 +84,11 @@ pub unsafe extern "C" fn memcpy(dst: *mut u8, src: *const u8, len: usize) -> *mu #[unsafe(no_mangle)] #[inline(never)] +/// C-compatible byte comparison for the analysis harness. +/// +/// # Safety +/// +/// `lhs` and `rhs` must each be readable for `len` bytes. pub unsafe extern "C" fn bcmp(lhs: *const u8, rhs: *const u8, len: usize) -> i32 { let mut acc = 0u8; let mut offset = 0usize; @@ -77,7 +98,7 @@ pub unsafe extern "C" fn bcmp(lhs: *const u8, rhs: *const u8, len: usize) -> i32 // SAFETY: C callers require both ranges to be readable for `len` bytes. let r = unsafe { ptr::read(rhs.add(offset)) }; acc |= l ^ r; - offset += 1; + offset = offset.strict_add(1); } i32::from(acc) } @@ -243,6 +264,12 @@ pub static mut CT_BINSEC_RSA_OTHER_32: [u8; 32] = [0u8; 32]; pub static mut CT_BINSEC_RESULT: u8 = 0; #[inline(always)] +/// Read a fixed-size byte array from a harness global. +/// +/// # Safety +/// +/// `ptr` must point to a valid, initialized array that remains readable for +/// the duration of the volatile read. unsafe fn array_from_global(ptr: *const [u8; N]) -> [u8; N] { // SAFETY: BINSEC harness globals are fixed-size byte arrays with static // storage duration. `read_volatile` keeps the symbolic bytes observable. @@ -250,6 +277,12 @@ unsafe fn array_from_global(ptr: *const [u8; N]) -> [u8; N] { } #[inline(always)] +/// Read a fixed-size limb array from a harness global. +/// +/// # Safety +/// +/// `ptr` must point to a valid, initialized array that remains readable for +/// the duration of the volatile read. unsafe fn limbs_from_global(ptr: *const [u64; N]) -> [u64; N] { // SAFETY: BINSEC harness globals are fixed-size limb arrays with static // storage duration. `read_volatile` keeps the symbolic limbs observable. @@ -555,8 +588,22 @@ macro_rules! aead_open_entry { }; } -aead_seal_entry!(ct_binsec_aes128gcm_seal, Aes128Gcm, Aes128GcmKey, Nonce96, CT_BINSEC_KEY_16, 0xA1); -aead_seal_entry!(ct_binsec_aes256gcm_seal, Aes256Gcm, Aes256GcmKey, Nonce96, CT_BINSEC_KEY_32, 0xA2); +aead_seal_entry!( + ct_binsec_aes128gcm_seal, + Aes128Gcm, + Aes128GcmKey, + Nonce96, + CT_BINSEC_KEY_16, + 0xA1 +); +aead_seal_entry!( + ct_binsec_aes256gcm_seal, + Aes256Gcm, + Aes256GcmKey, + Nonce96, + CT_BINSEC_KEY_32, + 0xA2 +); aead_seal_entry!( ct_binsec_aes128gcmsiv_seal, Aes128GcmSiv, @@ -589,7 +636,14 @@ aead_seal_entry!( CT_BINSEC_KEY_32, 0xA6 ); -aead_seal_entry!(ct_binsec_aegis256_seal, Aegis256, Aegis256Key, Nonce256, CT_BINSEC_KEY_32, 0xA7); +aead_seal_entry!( + ct_binsec_aegis256_seal, + Aegis256, + Aegis256Key, + Nonce256, + CT_BINSEC_KEY_32, + 0xA7 +); aead_seal_entry!( ct_binsec_ascon_aead128_seal, AsconAead128, @@ -599,8 +653,22 @@ aead_seal_entry!( 0xA8 ); -aead_open_entry!(ct_binsec_aes128gcm_open, Aes128Gcm, Aes128GcmKey, Nonce96, CT_BINSEC_KEY_16, 0xB1); -aead_open_entry!(ct_binsec_aes256gcm_open, Aes256Gcm, Aes256GcmKey, Nonce96, CT_BINSEC_KEY_32, 0xB2); +aead_open_entry!( + ct_binsec_aes128gcm_open, + Aes128Gcm, + Aes128GcmKey, + Nonce96, + CT_BINSEC_KEY_16, + 0xB1 +); +aead_open_entry!( + ct_binsec_aes256gcm_open, + Aes256Gcm, + Aes256GcmKey, + Nonce96, + CT_BINSEC_KEY_32, + 0xB2 +); aead_open_entry!( ct_binsec_aes128gcmsiv_open, Aes128GcmSiv, @@ -633,7 +701,14 @@ aead_open_entry!( CT_BINSEC_KEY_32, 0xB6 ); -aead_open_entry!(ct_binsec_aegis256_open, Aegis256, Aegis256Key, Nonce256, CT_BINSEC_KEY_32, 0xB7); +aead_open_entry!( + ct_binsec_aegis256_open, + Aegis256, + Aegis256Key, + Nonce256, + CT_BINSEC_KEY_32, + 0xB7 +); aead_open_entry!( ct_binsec_ascon_aead128_open, AsconAead128, @@ -658,7 +733,8 @@ pub extern "C" fn ct_binsec_argon2i_hash() -> ! { for word in dst { acc ^= word; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] @@ -726,7 +802,8 @@ pub extern "C" fn ct_binsec_curve25519_conditional_swap() -> ! { for limb in lhs.into_iter().chain(rhs) { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] @@ -740,7 +817,8 @@ pub extern "C" fn ct_binsec_ed25519_select_basepoint_cached() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] @@ -754,7 +832,8 @@ pub extern "C" fn ct_binsec_ecdsa_p256_select_signing_generator_affine() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] @@ -768,7 +847,8 @@ pub extern "C" fn ct_binsec_ecdsa_p384_select_signing_generator_affine() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] @@ -784,7 +864,8 @@ pub extern "C" fn ct_binsec_rsa_private_select_window_power_4() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] @@ -815,7 +896,9 @@ pub unsafe extern "C" fn ct_binsec_ed25519_select_basepoint_cached_avx2() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + ct_binsec_done( + (acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8, + ) } #[cfg(target_arch = "x86_64")] @@ -833,7 +916,9 @@ pub unsafe extern "C" fn ct_binsec_ed25519_select_basepoint_cached_ifma() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + ct_binsec_done( + (acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8, + ) } #[unsafe(no_mangle)] diff --git a/tools/ct-dudect/Cargo.lock b/tools/ct-dudect/Cargo.lock index 541d50d3..58b12237 100644 --- a/tools/ct-dudect/Cargo.lock +++ b/tools/ct-dudect/Cargo.lock @@ -396,18 +396,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", diff --git a/tools/ct-dudect/src/main.rs b/tools/ct-dudect/src/main.rs index f900b79d..479a2e95 100644 --- a/tools/ct-dudect/src/main.rs +++ b/tools/ct-dudect/src/main.rs @@ -1,21 +1,20 @@ use dudect_bencher::{BenchRng, Class, CtRunner, ctbench_main_with_seeds, rand::RngExt}; use rscrypto::aead::expert::AeadWithNonce; use rscrypto::{ - Aegis256, Aegis256Key, Aes128Gcm, Aes128GcmKey, Aes128GcmSiv, Aes128GcmSivKey, Aes256Gcm, Aes256GcmKey, - Aes256GcmSiv, Aes256GcmSivKey, Argon2Params, Argon2i, AsconAead128, AsconAead128Key, Blake2b256, Blake2b512, - Blake2bKey, Blake2s128, Blake2s256, Blake2sKey, Blake3, Blake3KeyedHash, ChaCha20Poly1305, ChaCha20Poly1305Key, - EcdsaP256SecretKey, EcdsaP384SecretKey, Ed25519Keypair, Ed25519SecretKey, HkdfSha256, - HkdfSha384, HmacSha256, HmacSha256Tag, HmacSha384, HmacSha384Tag, HmacSha512, HmacSha512Tag, Kmac256, MlKem512, - MlKem512Ciphertext, MlKem512DecapsulationKey, MlKem768, MlKem768Ciphertext, MlKem768DecapsulationKey, MlKem1024, - MlKem1024Ciphertext, MlKem1024DecapsulationKey, MlKemError, Pbkdf2Sha256, Pbkdf2Sha512, RsaOaepProfile, - RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, SecretBytes, Sha512, X25519SecretKey, XChaCha20Poly1305, - XChaCha20Poly1305Key, + Aegis256, Aegis256Key, Aes128Gcm, Aes128GcmKey, Aes128GcmSiv, Aes128GcmSivKey, Aes256Gcm, Aes256GcmKey, Aes256GcmSiv, + Aes256GcmSivKey, Argon2Params, Argon2i, AsconAead128, AsconAead128Key, Blake2b256, Blake2b512, Blake2bKey, + Blake2s128, Blake2s256, Blake2sKey, Blake3, Blake3KeyedHash, ChaCha20Poly1305, ChaCha20Poly1305Key, + EcdsaP256SecretKey, EcdsaP384SecretKey, Ed25519Keypair, Ed25519SecretKey, HkdfSha256, HkdfSha384, HmacSha256, + HmacSha256Tag, HmacSha384, HmacSha384Tag, HmacSha512, HmacSha512Tag, Kmac256, MlKem512, MlKem512Ciphertext, + MlKem512DecapsulationKey, MlKem768, MlKem768Ciphertext, MlKem768DecapsulationKey, MlKem1024, MlKem1024Ciphertext, + MlKem1024DecapsulationKey, MlKemError, Pbkdf2Sha256, Pbkdf2Sha512, RsaBlindingPair, RsaEncryptionError, + RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKeyPolicy, SecretBytes, Sha512, + X25519SecretKey, XChaCha20Poly1305, XChaCha20Poly1305Key, aead::{ - Nonce96, Nonce128, Nonce192, Nonce256, diag_aes128gcm_ctr32_be, diag_aes128gcm_ghash, - diag_aes128gcm_tag_aes, diag_aes128gcmsiv_ctr32, diag_aes128gcmsiv_derive_keys, - diag_aes128gcmsiv_polyval_digest, diag_aes128gcmsiv_raw_tag_aes, diag_aes256gcm_ctr32_be, - diag_aes256gcm_ghash, diag_aes256gcm_tag_aes, diag_aes256gcmsiv_ctr32, diag_aes256gcmsiv_derive_keys, - diag_aes256gcmsiv_raw_tag_aes, + Nonce96, Nonce128, Nonce192, Nonce256, diag_aes128gcm_ctr32_be, diag_aes128gcm_ghash, diag_aes128gcm_tag_aes, + diag_aes128gcmsiv_ctr32, diag_aes128gcmsiv_derive_keys, diag_aes128gcmsiv_polyval_digest, + diag_aes128gcmsiv_raw_tag_aes, diag_aes256gcm_ctr32_be, diag_aes256gcm_ghash, diag_aes256gcm_tag_aes, + diag_aes256gcmsiv_ctr32, diag_aes256gcmsiv_derive_keys, diag_aes256gcmsiv_raw_tag_aes, }, auth::{ diag_ecdsa_p256_basepoint_blinded_limb_digest, diag_ecdsa_p256_final_multiply_limb_digest, @@ -34,7 +33,6 @@ use rscrypto::{ diag_rsa_validate_pkcs8_private_key_der, diag_rsa_validate_pkcs8_private_key_der_stage, }, traits::Kem as _, - RsaEncryptionError, RsaPublicKeyPolicy, }; const DEFAULT_SAMPLES: usize = 20_000; @@ -76,9 +74,9 @@ fn balanced_classes(rng: &mut BenchRng, count: usize) -> Vec { let mut classes = Vec::with_capacity(count); while classes.len() < count { let start = classes.len(); - let len = (count - start).min(BLOCK_LEN); - let extra_left = len % 2 == 1 && rng.random::(); - let left_len = len / 2 + usize::from(extra_left); + let len = count.strict_sub(start).min(BLOCK_LEN); + let extra_left = len.rem_euclid(2) == 1 && rng.random::(); + let left_len = len.div_euclid(2).strict_add(usize::from(extra_left)); classes.extend((0..len).map(|index| if index < left_len { Class::Left } else { Class::Right })); for index in (1..len).rev() { @@ -92,7 +90,8 @@ fn balanced_classes(rng: &mut BenchRng, count: usize) -> Vec { fn mlkem_poly_from_seed(seed: u16) -> [u16; 256] { let mut out = [0u16; 256]; for (i, coeff) in out.iter_mut().enumerate() { - *coeff = seed.wrapping_add((i as u16).wrapping_mul(73)) % MLKEM_Q; + let index = u16::try_from(i).expect("ML-KEM polynomial indices fit u16"); + *coeff = seed.wrapping_add(index.wrapping_mul(73)).rem_euclid(MLKEM_Q); } out } @@ -100,7 +99,7 @@ fn mlkem_poly_from_seed(seed: u16) -> [u16; 256] { fn random_mlkem_poly(rng: &mut BenchRng) -> [u16; 256] { let mut out = [0u16; 256]; for coeff in &mut out { - *coeff = rng.random::() % MLKEM_Q; + *coeff = rng.random::().rem_euclid(MLKEM_Q); } out } @@ -126,24 +125,30 @@ fn random_mlkem_polyvec4(rng: &mut BenchRng) -> [[u16; 256]; 4] { fn json_string_value_n<'a>(json: &'a str, key: &str, n: usize) -> &'a str { let needle = format!("\"{key}\": \""); let mut rest = json; - for index in 0..=n { - let start = rest.find(&needle).unwrap_or_else(|| panic!("missing JSON string key {key} at index {index}")); - let value = &rest[start + needle.len()..]; - let end = value.find('"').unwrap_or_else(|| panic!("unterminated JSON string key {key} at index {index}")); - if index == n { - return &value[..end]; - } - rest = &value[end + 1..]; - } - unreachable!() -} - -fn hex_nibble(byte: u8) -> u8 { + for _ in 0..n { + let (_, value_and_rest) = rest + .split_once(&needle) + .expect("RSA fixture must contain the requested JSON string key"); + let (_, tail) = value_and_rest + .split_once('"') + .expect("RSA fixture JSON string must be terminated"); + rest = tail; + } + let (_, value_and_rest) = rest + .split_once(&needle) + .expect("RSA fixture must contain the requested JSON string key"); + let (value, _) = value_and_rest + .split_once('"') + .expect("RSA fixture JSON string must be terminated"); + value +} + +fn hex_nibble(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex byte"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } @@ -152,7 +157,11 @@ fn hex_to_vec(hex: &str) -> Vec { assert!(bytes.len().is_multiple_of(2), "hex string must have even length"); bytes .chunks_exact(2) - .map(|pair| (hex_nibble(pair[0]) << 4) | hex_nibble(pair[1])) + .map(|pair| { + let high = hex_nibble(pair[0]).expect("RSA fixture must contain hexadecimal bytes"); + let low = hex_nibble(pair[1]).expect("RSA fixture must contain hexadecimal bytes"); + high.strict_shl(4) | low + }) .collect() } @@ -162,51 +171,59 @@ fn rsa_pkcs8_der(index: usize) -> Vec { fn rsa_ct_fixture_key(index: usize) -> RsaPrivateKey { let der = rsa_pkcs8_der(index); - RsaPrivateKey::from_pkcs8_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()).unwrap() + RsaPrivateKey::from_pkcs8_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("bundled RSA fixture must contain a valid private key") } fn rsa_blinding_pair(key: &RsaPrivateKey) -> (Vec, Vec) { let modulus = key.public_key().modulus(); let mut factor = vec![0u8; modulus.len()]; - factor[modulus.len() - 1] = 2; + factor[modulus.len().strict_sub(1)] = 2; let mut plus_one = modulus.to_vec(); let mut carry = 1u16; for byte in plus_one.iter_mut().rev() { - let sum = u16::from(*byte) + carry; - *byte = sum as u8; + let sum = u16::from(*byte).strict_add(carry); + *byte = sum.to_le_bytes()[0]; carry = sum >> 8; if carry == 0 { break; } } if carry != 0 { - plus_one.insert(0, carry as u8); + plus_one.insert(0, carry.to_le_bytes()[0]); } let mut quotient = Vec::with_capacity(plus_one.len()); let mut remainder = 0u16; for byte in plus_one { let value = (remainder << 8) | u16::from(byte); - quotient.push((value / 2) as u8); - remainder = value % 2; + quotient.push(value.div_euclid(2).to_le_bytes()[0]); + remainder = value.rem_euclid(2); } - let first_nonzero = quotient.iter().position(|&byte| byte != 0).unwrap_or(quotient.len() - 1); - let inverse = "ient[first_nonzero..]; + let first_nonzero = quotient + .iter() + .position(|&byte| byte != 0) + .unwrap_or_else(|| quotient.len().strict_sub(1)); + let inverse = quotient + .get(first_nonzero..) + .expect("the first nonzero quotient byte is in bounds"); let mut inverse_fixed = vec![0u8; modulus.len()]; - inverse_fixed[modulus.len() - inverse.len()..].copy_from_slice(inverse); + let inverse_start = modulus.len().strict_sub(inverse.len()); + inverse_fixed + .get_mut(inverse_start..) + .expect("the RSA inverse fits the modulus width") + .copy_from_slice(inverse); (factor, inverse_fixed) } fn argon2i_params() -> Argon2Params { - Argon2Params::new(32, 1, 1) - .unwrap() + Argon2Params::new(32, 1, 1).expect("fixed Argon2i profile is valid") } fn argon2i_parallel_params() -> Argon2Params { - Argon2Params::new(512, 1, 4) - .unwrap() + Argon2Params::new(512, 1, 4).expect("fixed parallel Argon2i profile is valid") } macro_rules! fixed_owner_eq_case { @@ -224,7 +241,11 @@ macro_rules! fixed_owner_eq_case { } for (class, left, right) in inputs { - runner.run_one(class, || $entry(left.as_ptr(), right.as_ptr()) == 1); + runner.run_one(class, || { + // SAFETY: left and right are initialized [$len; u8] values whose storage remains alive + // and immutable for the entrypoint's fixed-size reads. + unsafe { $entry(left.as_ptr(), right.as_ptr()) == 1 } + }); } } }; @@ -303,7 +324,10 @@ fn hmac_sha256_truncated_64_valid_vs_invalid_tag(runner: &mut CtRunner, rng: &mu let class = random_class(rng); let key = rand_array::<32>(rng); let full_tag = HmacSha256::mac(&key, MESSAGE); - let mut expected = *full_tag.as_bytes().first_chunk::<8>().unwrap(); + let mut expected = *full_tag + .as_bytes() + .first_chunk::<8>() + .expect("HMAC-SHA-256 tags contain eight bytes"); if matches!(class, Class::Right) { expected[0] ^= 1; } @@ -311,7 +335,9 @@ fn hmac_sha256_truncated_64_valid_vs_invalid_tag(runner: &mut CtRunner, rng: &mu } for (class, key, expected) in inputs { - runner.run_one(class, || HmacSha256::verify_truncated_tag_64(&key, MESSAGE, &expected).is_ok()); + runner.run_one(class, || { + HmacSha256::verify_truncated_tag_64(&key, MESSAGE, &expected).is_ok() + }); } } @@ -365,7 +391,9 @@ macro_rules! aead_fixed_vs_random_key_open { let key = <$key>::from_bytes(key); let cipher = <$cipher>::new(&key); let mut ciphertext = AEAD_PLAINTEXT; - let tag = cipher.encrypt_in_place(&nonce, AAD, &mut ciphertext).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, AAD, &mut ciphertext) + .expect("fixed-size AEAD timing fixture must encrypt"); inputs.push((class, key, nonce, ciphertext, tag)); } @@ -502,7 +530,9 @@ fn gcm_siv_diag_polyval_fixed_vs_random_auth_key(runner: &mut CtRunner, rng: &mu } for (class, auth_key) in inputs { - runner.run_one(class, || diag_aes128gcmsiv_polyval_digest(&auth_key, AAD, &AEAD_PLAINTEXT)); + runner.run_one(class, || { + diag_aes128gcmsiv_polyval_digest(&auth_key, AAD, &AEAD_PLAINTEXT) + }); } } @@ -766,7 +796,7 @@ macro_rules! mlkem_dudect_profile { } for (class, sigma) in inputs { - runner.run_one(class, || std::hint::black_box($diag(rho, sigma))[0]); + runner.run_one(class, || core::hint::black_box($diag(rho, sigma))[0]); } } @@ -775,7 +805,7 @@ macro_rules! mlkem_dudect_profile { out.copy_from_slice(&[0x51; <$profile>::KEY_GENERATION_RANDOM_SIZE]); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("fixed ML-KEM key-generation entropy must produce a keypair"); let mut inputs = Vec::with_capacity(samples()); for _ in 0..samples() { let class = random_class(rng); @@ -793,8 +823,8 @@ macro_rules! mlkem_dudect_profile { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap(); - std::hint::black_box(ciphertext.as_bytes()[0] ^ shared_secret.as_bytes()[0]) + .expect("fixed ML-KEM encapsulation entropy must produce a ciphertext"); + core::hint::black_box(ciphertext.as_bytes()[0] ^ shared_secret.as_bytes()[0]) }); } } @@ -804,12 +834,12 @@ macro_rules! mlkem_dudect_profile { out.copy_from_slice(&[0x61; <$profile>::KEY_GENERATION_RANDOM_SIZE]); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("fixed ML-KEM key-generation entropy must produce a keypair"); let (ciphertext, _) = <$profile>::encapsulate(&fixed_encapsulation_key, |out| { out.copy_from_slice(&[0x62; <$profile>::ENCAPSULATION_RANDOM_SIZE]); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("fixed ML-KEM encapsulation entropy must produce a ciphertext"); let secret_key_len = <$profile>::DECAPSULATION_KEY_SIZE - <$profile>::ENCAPSULATION_KEY_SIZE - 64; let mut inputs = Vec::with_capacity(samples()); for _ in 0..samples() { @@ -821,7 +851,7 @@ macro_rules! mlkem_dudect_profile { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap() + .expect("random ML-KEM key-generation entropy must produce a keypair") .1; decapsulation_key_bytes[..secret_key_len] .copy_from_slice(&random_decapsulation_key.as_bytes()[..secret_key_len]); @@ -831,8 +861,9 @@ macro_rules! mlkem_dudect_profile { for (class, decapsulation_key) in inputs { runner.run_one(class, || { - let shared_secret = <$profile>::decapsulate(&decapsulation_key, &ciphertext).unwrap(); - std::hint::black_box(shared_secret.as_bytes()[0]) + let shared_secret = <$profile>::decapsulate(&decapsulation_key, &ciphertext) + .expect("well-formed ML-KEM ciphertext must decapsulate"); + core::hint::black_box(shared_secret.as_bytes()[0]) }); } } @@ -842,12 +873,12 @@ macro_rules! mlkem_dudect_profile { out.copy_from_slice(&[0x71; <$profile>::KEY_GENERATION_RANDOM_SIZE]); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("fixed ML-KEM key-generation entropy must produce a keypair"); let (ciphertext, _) = <$profile>::encapsulate(&encapsulation_key, |out| { out.copy_from_slice(&[0x72; <$profile>::ENCAPSULATION_RANDOM_SIZE]); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("fixed ML-KEM encapsulation entropy must produce a ciphertext"); let mut rejected_ciphertext = ciphertext.to_bytes(); rejected_ciphertext[0] ^= 1; let rejected_ciphertext = <$ciphertext>::from_bytes(rejected_ciphertext); @@ -868,8 +899,9 @@ macro_rules! mlkem_dudect_profile { for (class, decapsulation_key) in inputs { runner.run_one(class, || { - let shared_secret = <$profile>::decapsulate(&decapsulation_key, &rejected_ciphertext).unwrap(); - std::hint::black_box(shared_secret.as_bytes()[0]) + let shared_secret = <$profile>::decapsulate(&decapsulation_key, &rejected_ciphertext) + .expect("ML-KEM implicit rejection must produce a shared secret"); + core::hint::black_box(shared_secret.as_bytes()[0]) }); } } @@ -920,7 +952,7 @@ fn mlkem_arithmetic_ntt_fixed_vs_random_poly(runner: &mut CtRunner, rng: &mut Be } for (class, poly) in inputs { - runner.run_one(class, || std::hint::black_box(diag_mlkem_ntt_input_digest(poly))); + runner.run_one(class, || core::hint::black_box(diag_mlkem_ntt_input_digest(poly))); } } @@ -937,7 +969,9 @@ fn mlkem_arithmetic_inverse_ntt_fixed_vs_random_poly(runner: &mut CtRunner, rng: } for (class, poly) in inputs { - runner.run_one(class, || std::hint::black_box(diag_mlkem_inverse_ntt_montgomery_product_input_digest(poly))); + runner.run_one(class, || { + core::hint::black_box(diag_mlkem_inverse_ntt_montgomery_product_input_digest(poly)) + }); } } @@ -954,7 +988,9 @@ fn mlkem_arithmetic_to_product_domain_fixed_vs_random_poly(runner: &mut CtRunner } for (class, poly) in inputs { - runner.run_one(class, || std::hint::black_box(diag_mlkem_to_montgomery_product_domain_input_digest(poly))); + runner.run_one(class, || { + core::hint::black_box(diag_mlkem_to_montgomery_product_domain_input_digest(poly)) + }); } } @@ -971,7 +1007,9 @@ fn mlkem_arithmetic_from_product_domain_fixed_vs_random_poly(runner: &mut CtRunn } for (class, poly) in inputs { - runner.run_one(class, || std::hint::black_box(diag_mlkem_from_montgomery_product_domain_input_digest(poly))); + runner.run_one(class, || { + core::hint::black_box(diag_mlkem_from_montgomery_product_domain_input_digest(poly)) + }); } } @@ -992,7 +1030,9 @@ fn mlkem_arithmetic_basemul_fixed_vs_random_operands(runner: &mut CtRunner, rng: } for (class, a, b, acc) in inputs { - runner.run_one(class, || std::hint::black_box(diag_mlkem_multiply_ntts_add_assign_input_digest(a, b, acc))); + runner.run_one(class, || { + core::hint::black_box(diag_mlkem_multiply_ntts_add_assign_input_digest(a, b, acc)) + }); } } @@ -1007,13 +1047,19 @@ fn mlkem1024_arithmetic_dot_fixed_vs_random_operands(runner: &mut CtRunner, rng: mlkem_poly_from_seed(0xA01), ) } else { - (random_mlkem_polyvec4(rng), random_mlkem_polyvec4(rng), random_mlkem_poly(rng)) + ( + random_mlkem_polyvec4(rng), + random_mlkem_polyvec4(rng), + random_mlkem_poly(rng), + ) }; inputs.push((class, a, b, acc)); } for (class, a, b, acc) in inputs { - runner.run_one(class, || std::hint::black_box(diag_mlkem1024_multiply_ntts_accumulate_input_digest(a, b, acc))); + runner.run_one(class, || { + core::hint::black_box(diag_mlkem1024_multiply_ntts_accumulate_input_digest(a, b, acc)) + }); } } @@ -1079,7 +1125,10 @@ fn ed25519_keypair_sign_fixed_vs_random_secret(runner: &mut CtRunner, rng: &mut } else { rand_array::<{ Ed25519SecretKey::LENGTH }>(rng) }; - inputs.push((class, Ed25519Keypair::from_secret_key(Ed25519SecretKey::from_bytes(secret)))); + inputs.push(( + class, + Ed25519Keypair::from_secret_key(Ed25519SecretKey::from_bytes(secret)), + )); } for (class, keypair) in inputs { @@ -1117,9 +1166,14 @@ fn ecdsa_p256_sign_fixed_vs_random_secret(runner: &mut CtRunner, rng: &mut Bench } for (class, secret, blind) in inputs { - let key = EcdsaP256SecretKey::from_bytes(secret).unwrap(); + let key = EcdsaP256SecretKey::from_bytes(secret).expect("generated P-256 secret scalar is valid"); runner.run_one(class, || { - std::hint::black_box(key.try_sign_blinded(MESSAGE, |out| out.copy_from_slice(&blind)).unwrap().to_bytes()); + core::hint::black_box( + key + .try_sign_blinded(MESSAGE, |out| out.copy_from_slice(&blind)) + .expect("valid P-256 key and blinding input must sign") + .to_bytes(), + ); }); } } @@ -1136,9 +1190,14 @@ fn ecdsa_p384_sign_fixed_vs_random_secret(runner: &mut CtRunner, rng: &mut Bench } for (class, secret, blind) in inputs { - let key = EcdsaP384SecretKey::from_bytes(secret).unwrap(); + let key = EcdsaP384SecretKey::from_bytes(secret).expect("generated P-384 secret scalar is valid"); runner.run_one(class, || { - std::hint::black_box(key.try_sign_blinded(MESSAGE, |out| out.copy_from_slice(&blind)).unwrap().to_bytes()); + core::hint::black_box( + key + .try_sign_blinded(MESSAGE, |out| out.copy_from_slice(&blind)) + .expect("valid P-384 key and blinding input must sign") + .to_bytes(), + ); }); } } @@ -1156,7 +1215,7 @@ fn ecdsa_p256_diag_nonce_reduce_fixed_vs_random_secret(runner: &mut CtRunner, rn for (class, secret) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_nonce_reduce_limb_digest(secret, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p256_nonce_reduce_limb_digest(secret, MESSAGE))[0] }); } } @@ -1174,7 +1233,7 @@ fn ecdsa_p256_diag_reduce_wide_fixed_vs_random_input(runner: &mut CtRunner, rng: for (class, wide) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_reduce_wide_order_limb_digest(wide))[0] + core::hint::black_box(diag_ecdsa_p256_reduce_wide_order_limb_digest(wide))[0] }); } } @@ -1192,7 +1251,7 @@ fn ecdsa_p256_diag_basepoint_blinded_fixed_vs_random_secret(runner: &mut CtRunne for (class, secret, blind) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_basepoint_blinded_limb_digest(secret, blind, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p256_basepoint_blinded_limb_digest(secret, blind, MESSAGE))[0] }); } } @@ -1210,7 +1269,7 @@ fn ecdsa_p256_diag_scalar_finish_fixed_vs_random_secret(runner: &mut CtRunner, r for (class, secret, nonce_wide) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_scalar_finish_limb_digest(secret, nonce_wide, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p256_scalar_finish_limb_digest(secret, nonce_wide, MESSAGE))[0] }); } } @@ -1228,7 +1287,7 @@ fn ecdsa_p256_diag_order_mul_fixed_r_fixed_vs_random_secret(runner: &mut CtRunne for (class, secret) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_order_mul_fixed_r_limb_digest(secret))[0] + core::hint::black_box(diag_ecdsa_p256_order_mul_fixed_r_limb_digest(secret))[0] }); } } @@ -1246,7 +1305,7 @@ fn ecdsa_p256_diag_blinded_order_mul_fixed_vs_random_secret(runner: &mut CtRunne for (class, secret, blind) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_order_mul_blinded_fixed_r_limb_digest(secret, blind))[0] + core::hint::black_box(diag_ecdsa_p256_order_mul_blinded_fixed_r_limb_digest(secret, blind))[0] }); } } @@ -1264,7 +1323,7 @@ fn ecdsa_p256_diag_nonce_inverse_fixed_vs_random_secret(runner: &mut CtRunner, r for (class, secret) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_nonce_inverse_limb_digest(secret, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p256_nonce_inverse_limb_digest(secret, MESSAGE))[0] }); } } @@ -1282,7 +1341,7 @@ fn ecdsa_p256_diag_final_multiply_fixed_vs_random_secret(runner: &mut CtRunner, for (class, secret, nonce_wide) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_final_multiply_limb_digest(secret, nonce_wide, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p256_final_multiply_limb_digest(secret, nonce_wide, MESSAGE))[0] }); } } @@ -1300,7 +1359,7 @@ fn ecdsa_p384_diag_nonce_reduce_fixed_vs_random_secret(runner: &mut CtRunner, rn for (class, secret) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_nonce_reduce_limb_digest(secret, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p384_nonce_reduce_limb_digest(secret, MESSAGE))[0] }); } } @@ -1318,7 +1377,7 @@ fn ecdsa_p384_diag_reduce_wide_fixed_vs_random_input(runner: &mut CtRunner, rng: for (class, wide) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_reduce_wide_order_limb_digest(wide))[0] + core::hint::black_box(diag_ecdsa_p384_reduce_wide_order_limb_digest(wide))[0] }); } } @@ -1336,7 +1395,7 @@ fn ecdsa_p384_diag_basepoint_blinded_fixed_vs_random_secret(runner: &mut CtRunne for (class, secret, blind) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_basepoint_blinded_limb_digest(secret, blind, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p384_basepoint_blinded_limb_digest(secret, blind, MESSAGE))[0] }); } } @@ -1354,7 +1413,7 @@ fn ecdsa_p384_diag_scalar_finish_fixed_vs_random_secret(runner: &mut CtRunner, r for (class, secret, nonce_wide) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_scalar_finish_limb_digest(secret, nonce_wide, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p384_scalar_finish_limb_digest(secret, nonce_wide, MESSAGE))[0] }); } } @@ -1372,7 +1431,7 @@ fn ecdsa_p384_diag_order_mul_fixed_r_fixed_vs_random_secret(runner: &mut CtRunne for (class, secret) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_order_mul_fixed_r_limb_digest(secret))[0] + core::hint::black_box(diag_ecdsa_p384_order_mul_fixed_r_limb_digest(secret))[0] }); } } @@ -1390,7 +1449,7 @@ fn ecdsa_p384_diag_nonce_inverse_fixed_vs_random_secret(runner: &mut CtRunner, r for (class, secret) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_nonce_inverse_limb_digest(secret, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p384_nonce_inverse_limb_digest(secret, MESSAGE))[0] }); } } @@ -1408,7 +1467,7 @@ fn ecdsa_p384_diag_final_multiply_fixed_vs_random_secret(runner: &mut CtRunner, for (class, secret, nonce_wide) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_final_multiply_limb_digest(secret, nonce_wide, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p384_final_multiply_limb_digest(secret, nonce_wide, MESSAGE))[0] }); } } @@ -1436,8 +1495,7 @@ fn rsa_pkcs1v15_fixed_vs_random_message(runner: &mut CtRunner, rng: &mut BenchRn .sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, &message, - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), &mut out, ) .is_ok() @@ -1470,8 +1528,7 @@ fn rsa_pss_fixed_vs_random_message(runner: &mut CtRunner, rng: &mut BenchRng) { RsaPssProfile::Sha256, &message, &salt, - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), &mut out, ) .is_ok() @@ -1498,8 +1555,7 @@ fn rsa_pkcs1v15_full_width_vs_short_canonical_crt_exponent(runner: &mut CtRunner .sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, &message, - blinding_factor, - blinding_inverse, + RsaBlindingPair::new(blinding_factor, blinding_inverse), &mut out, ) .is_ok() @@ -1532,7 +1588,7 @@ fn rsa_oaep_decrypt_fixed_vs_random_plaintext(runner: &mut CtRunner, rng: &mut B out.copy_from_slice(&seed); Ok(()) }) - .unwrap(); + .expect("fixed OAEP fixture must encrypt"); inputs.push((class, ciphertext)); } @@ -1544,8 +1600,7 @@ fn rsa_oaep_decrypt_fixed_vs_random_plaintext(runner: &mut CtRunner, rng: &mut B RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), &mut out, ) .is_ok() @@ -1573,7 +1628,7 @@ fn rsa_pkcs1v15_decrypt_fixed_vs_random_plaintext(runner: &mut CtRunner, rng: &m out.fill(0x5d); Ok(()) }) - .unwrap(); + .expect("fixed PKCS#1 v1.5 fixture must encrypt"); inputs.push((class, ciphertext)); } @@ -1581,7 +1636,11 @@ fn rsa_pkcs1v15_decrypt_fixed_vs_random_plaintext(runner: &mut CtRunner, rng: &m runner.run_one(class, || { let mut out = vec![0u8; sig_len]; key - .decrypt_pkcs1v15_with_blinding_factor(&ciphertext, &blinding_factor, &blinding_inverse, &mut out) + .decrypt_pkcs1v15_with_blinding_factor( + &ciphertext, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), + &mut out, + ) .is_ok() }); } @@ -1612,7 +1671,9 @@ fn rsa_private_component_validation_fixed_vs_random_component(runner: &mut CtRun } for (class, component) in inputs { - runner.run_one(class, || diag_rsa_private_component_validation_32(&component, &upper_bound, &other)); + runner.run_one(class, || { + diag_rsa_private_component_validation_32(&component, &upper_bound, &other) + }); } } @@ -1629,7 +1690,8 @@ fn rsa_private_key_pkcs8_import_key_a_vs_key_b(runner: &mut CtRunner, rng: &mut let selected = if matches!(class, Class::Left) { &der_a } else { &der_b }; der.copy_from_slice(selected); runner.run_one(class, || { - let key = RsaPrivateKey::from_pkcs8_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let key = RsaPrivateKey::from_pkcs8_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("selected bundled RSA fixture must import"); key.signature_len() }); } @@ -1716,7 +1778,10 @@ fn rsa_private_key_pkcs8_import_stage_key_a_vs_key_b(runner: &mut CtRunner, rng: for class in inputs { let selected = if matches!(class, Class::Left) { &der_a } else { &der_b }; der.copy_from_slice(selected); - runner.run_one(class, || diag_rsa_import_pkcs8_private_key_der_stage(&der, &policy, stage).unwrap()); + runner.run_one(class, || { + diag_rsa_import_pkcs8_private_key_der_stage(&der, &policy, stage) + .expect("selected bundled RSA fixture must reach the requested import stage") + }); } } @@ -1735,9 +1800,10 @@ fn rsa_private_key_pkcs8_validate_stage_key_a_vs_key_b(runner: &mut CtRunner, rn der.copy_from_slice(selected); runner.run_one(class, || { if stage == u8::MAX { - diag_rsa_validate_pkcs8_private_key_der(&der, &policy).unwrap() + diag_rsa_validate_pkcs8_private_key_der(&der, &policy).expect("selected bundled RSA fixture must validate") } else { - diag_rsa_validate_pkcs8_private_key_der_stage(&der, &policy, stage).unwrap() + diag_rsa_validate_pkcs8_private_key_der_stage(&der, &policy, stage) + .expect("selected bundled RSA fixture must reach the requested validation stage") } }); } @@ -1962,7 +2028,7 @@ macro_rules! blake2_keyed_fixed_vs_random { } for (class, key) in inputs { - let key = <$key_ty>::new(&key).unwrap(); + let key = <$key_ty>::new(&key).expect("32-byte BLAKE2 timing key is valid"); runner.run_one(class, || <$ty>::keyed_digest(key, MESSAGE)[0]); } } @@ -2040,10 +2106,7 @@ ctbench_main_with_seeds!( (owner_eq_64_equal_vs_first_diff, Some(0x6f776e657236345f)), (secret_wrappers_debug_fixed_vs_random, Some(0x7365637265745f77)), (hmac_sha256_valid_vs_invalid_tag, Some(0x686d61635f736861)), - ( - hmac_sha256_truncated_64_valid_vs_invalid_tag, - Some(0x686d61635f743634) - ), + (hmac_sha256_truncated_64_valid_vs_invalid_tag, Some(0x686d61635f743634)), (hmac_sha384_valid_vs_invalid_tag, Some(0x686d61633338345f)), (hmac_sha512_valid_vs_invalid_tag, Some(0x686d61633531325f)), (kmac256_valid_vs_invalid_tag, Some(0x6b6d61633235365f)), @@ -2055,8 +2118,14 @@ ctbench_main_with_seeds!( (aes128_gcm_siv_diag_derive_fixed_vs_random_key, Some(0x6131323867646572)), (aes256_gcm_siv_diag_derive_fixed_vs_random_key, Some(0x6132353667646572)), (gcm_siv_diag_polyval_fixed_vs_random_auth_key, Some(0x6763736976706f6c)), - (aes128_gcm_siv_diag_raw_tag_aes_fixed_vs_random_key, Some(0x6131323867746167)), - (aes256_gcm_siv_diag_raw_tag_aes_fixed_vs_random_key, Some(0x6132353667746167)), + ( + aes128_gcm_siv_diag_raw_tag_aes_fixed_vs_random_key, + Some(0x6131323867746167) + ), + ( + aes256_gcm_siv_diag_raw_tag_aes_fixed_vs_random_key, + Some(0x6132353667746167) + ), (aes128_gcm_diag_ctr32_be_fixed_vs_random_key, Some(0x6731323863747262)), (aes256_gcm_diag_ctr32_be_fixed_vs_random_key, Some(0x6732353663747262)), (aes128_gcm_diag_ghash_fixed_vs_random_h, Some(0x6731323867686173)), @@ -2074,43 +2143,124 @@ ctbench_main_with_seeds!( (x25519_fixed_vs_random_scalar, Some(0x7832353531395f63)), (mlkem512_keygen_secret_noise_fixed_vs_random, Some(0x6d6b3531326b676e)), (mlkem512_encapsulate_fixed_vs_random_coins, Some(0x6d6b353132656e63)), - (mlkem512_decapsulate_fixed_vs_random_secret_key, Some(0x6d6b353132646563)), - (mlkem512_decapsulate_rejection_seed_fixed_vs_random, Some(0x6d6b35313272656a)), + ( + mlkem512_decapsulate_fixed_vs_random_secret_key, + Some(0x6d6b353132646563) + ), + ( + mlkem512_decapsulate_rejection_seed_fixed_vs_random, + Some(0x6d6b35313272656a) + ), (mlkem768_keygen_secret_noise_fixed_vs_random, Some(0x6d6c6b656d6b676e)), (mlkem768_encapsulate_fixed_vs_random_coins, Some(0x6d6c6b656d656e63)), - (mlkem768_decapsulate_fixed_vs_random_secret_key, Some(0x6d6c6b656d646563)), - (mlkem768_decapsulate_rejection_seed_fixed_vs_random, Some(0x6d6c6b656d72656a)), + ( + mlkem768_decapsulate_fixed_vs_random_secret_key, + Some(0x6d6c6b656d646563) + ), + ( + mlkem768_decapsulate_rejection_seed_fixed_vs_random, + Some(0x6d6c6b656d72656a) + ), (mlkem1024_keygen_secret_noise_fixed_vs_random, Some(0x6d6b313032346b67)), (mlkem1024_encapsulate_fixed_vs_random_coins, Some(0x6d6b31303234656e)), - (mlkem1024_decapsulate_fixed_vs_random_secret_key, Some(0x6d6b313032346465)), - (mlkem1024_decapsulate_rejection_seed_fixed_vs_random, Some(0x6d6b31303234726a)), + ( + mlkem1024_decapsulate_fixed_vs_random_secret_key, + Some(0x6d6b313032346465) + ), + ( + mlkem1024_decapsulate_rejection_seed_fixed_vs_random, + Some(0x6d6b31303234726a) + ), (mlkem_arithmetic_ntt_fixed_vs_random_poly, Some(0x6d6c6b6e7474706f)), - (mlkem_arithmetic_inverse_ntt_fixed_vs_random_poly, Some(0x6d6c6b696e747470)), - (mlkem_arithmetic_to_product_domain_fixed_vs_random_poly, Some(0x6d6c6b746f70726f)), - (mlkem_arithmetic_from_product_domain_fixed_vs_random_poly, Some(0x6d6c6b667270726f)), - (mlkem_arithmetic_basemul_fixed_vs_random_operands, Some(0x6d6c6b626173656d)), - (mlkem1024_arithmetic_dot_fixed_vs_random_operands, Some(0x6d6c6b313032646f)), + ( + mlkem_arithmetic_inverse_ntt_fixed_vs_random_poly, + Some(0x6d6c6b696e747470) + ), + ( + mlkem_arithmetic_to_product_domain_fixed_vs_random_poly, + Some(0x6d6c6b746f70726f) + ), + ( + mlkem_arithmetic_from_product_domain_fixed_vs_random_poly, + Some(0x6d6c6b667270726f) + ), + ( + mlkem_arithmetic_basemul_fixed_vs_random_operands, + Some(0x6d6c6b626173656d) + ), + ( + mlkem1024_arithmetic_dot_fixed_vs_random_operands, + Some(0x6d6c6b313032646f) + ), (ed25519_sign_fixed_vs_random_secret, Some(0x656432353531395f)), (ed25519_public_key_fixed_vs_random_secret, Some(0x6564323535313950)), - (ed25519_sha512_secret_expand_fixed_vs_random_secret, Some(0x6564323535314853)), + ( + ed25519_sha512_secret_expand_fixed_vs_random_secret, + Some(0x6564323535314853) + ), (ed25519_keypair_sign_fixed_vs_random_secret, Some(0x656432353531394b)), (ecdsa_p256_sign_fixed_vs_random_secret, Some(0x703235365f736967)), (ecdsa_p384_sign_fixed_vs_random_secret, Some(0x703338345f736967)), - (ecdsa_p256_diag_nonce_reduce_fixed_vs_random_secret, Some(0x703235366e6f6e63)), - (ecdsa_p256_diag_reduce_wide_fixed_vs_random_input, Some(0x7032353672656475)), - (ecdsa_p256_diag_basepoint_blinded_fixed_vs_random_secret, Some(0x7032353662617365)), - (ecdsa_p256_diag_scalar_finish_fixed_vs_random_secret, Some(0x7032353666696e73)), - (ecdsa_p256_diag_order_mul_fixed_r_fixed_vs_random_secret, Some(0x703235366d756c72)), - (ecdsa_p256_diag_blinded_order_mul_fixed_vs_random_secret, Some(0x703235366d756c62)), - (ecdsa_p256_diag_nonce_inverse_fixed_vs_random_secret, Some(0x70323536696e766b)), - (ecdsa_p256_diag_final_multiply_fixed_vs_random_secret, Some(0x703235366d756c73)), - (ecdsa_p384_diag_nonce_reduce_fixed_vs_random_secret, Some(0x703338346e6f6e63)), - (ecdsa_p384_diag_reduce_wide_fixed_vs_random_input, Some(0x7033383472656475)), - (ecdsa_p384_diag_basepoint_blinded_fixed_vs_random_secret, Some(0x7033383462617365)), - (ecdsa_p384_diag_scalar_finish_fixed_vs_random_secret, Some(0x7033383466696e73)), - (ecdsa_p384_diag_order_mul_fixed_r_fixed_vs_random_secret, Some(0x703338346d756c72)), - (ecdsa_p384_diag_nonce_inverse_fixed_vs_random_secret, Some(0x70333834696e766b)), - (ecdsa_p384_diag_final_multiply_fixed_vs_random_secret, Some(0x703338346d756c73)), + ( + ecdsa_p256_diag_nonce_reduce_fixed_vs_random_secret, + Some(0x703235366e6f6e63) + ), + ( + ecdsa_p256_diag_reduce_wide_fixed_vs_random_input, + Some(0x7032353672656475) + ), + ( + ecdsa_p256_diag_basepoint_blinded_fixed_vs_random_secret, + Some(0x7032353662617365) + ), + ( + ecdsa_p256_diag_scalar_finish_fixed_vs_random_secret, + Some(0x7032353666696e73) + ), + ( + ecdsa_p256_diag_order_mul_fixed_r_fixed_vs_random_secret, + Some(0x703235366d756c72) + ), + ( + ecdsa_p256_diag_blinded_order_mul_fixed_vs_random_secret, + Some(0x703235366d756c62) + ), + ( + ecdsa_p256_diag_nonce_inverse_fixed_vs_random_secret, + Some(0x70323536696e766b) + ), + ( + ecdsa_p256_diag_final_multiply_fixed_vs_random_secret, + Some(0x703235366d756c73) + ), + ( + ecdsa_p384_diag_nonce_reduce_fixed_vs_random_secret, + Some(0x703338346e6f6e63) + ), + ( + ecdsa_p384_diag_reduce_wide_fixed_vs_random_input, + Some(0x7033383472656475) + ), + ( + ecdsa_p384_diag_basepoint_blinded_fixed_vs_random_secret, + Some(0x7033383462617365) + ), + ( + ecdsa_p384_diag_scalar_finish_fixed_vs_random_secret, + Some(0x7033383466696e73) + ), + ( + ecdsa_p384_diag_order_mul_fixed_r_fixed_vs_random_secret, + Some(0x703338346d756c72) + ), + ( + ecdsa_p384_diag_nonce_inverse_fixed_vs_random_secret, + Some(0x70333834696e766b) + ), + ( + ecdsa_p384_diag_final_multiply_fixed_vs_random_secret, + Some(0x703338346d756c73) + ), (rsa_pkcs1v15_fixed_vs_random_message, Some(0x7273615f7369676e)), (rsa_pss_fixed_vs_random_message, Some(0x7273615f70737373)), ( @@ -2119,25 +2269,76 @@ ctbench_main_with_seeds!( ), (rsa_oaep_decrypt_fixed_vs_random_plaintext, Some(0x7273615f6f616570)), (rsa_pkcs1v15_decrypt_fixed_vs_random_plaintext, Some(0x7273615f64656331)), - (rsa_private_component_validation_fixed_vs_random_component, Some(0x7273615f636f6d70)), + ( + rsa_private_component_validation_fixed_vs_random_component, + Some(0x7273615f636f6d70) + ), (rsa_private_key_pkcs8_import_key_a_vs_key_b, Some(0x7273615f6b657969)), (rsa_private_key_pkcs8_validate_key_a_vs_key_b, Some(0x7273615f76616c69)), - (rsa_private_key_pkcs8_validate_stage0_key_a_vs_key_b, Some(0x7273615f76733030)), - (rsa_private_key_pkcs8_validate_stage1_key_a_vs_key_b, Some(0x7273615f76733031)), - (rsa_private_key_pkcs8_validate_stage2_key_a_vs_key_b, Some(0x7273615f76733032)), - (rsa_private_key_pkcs8_validate_stage3_key_a_vs_key_b, Some(0x7273615f76733033)), - (rsa_private_key_pkcs8_validate_stage4_key_a_vs_key_b, Some(0x7273615f76733034)), - (rsa_private_key_pkcs8_validate_stage30_key_a_vs_key_b, Some(0x7273615f76333030)), - (rsa_private_key_pkcs8_validate_stage31_key_a_vs_key_b, Some(0x7273615f76333031)), - (rsa_private_key_pkcs8_validate_stage32_key_a_vs_key_b, Some(0x7273615f76333032)), - (rsa_private_key_pkcs8_validate_stage40_key_a_vs_key_b, Some(0x7273615f76343030)), - (rsa_private_key_pkcs8_validate_stage41_key_a_vs_key_b, Some(0x7273615f76343031)), - (rsa_private_key_pkcs8_validate_stage42_key_a_vs_key_b, Some(0x7273615f76343032)), - (rsa_private_key_pkcs8_import_stage50_key_a_vs_key_b, Some(0x7273615f69353030)), - (rsa_private_key_pkcs8_import_stage51_key_a_vs_key_b, Some(0x7273615f69353031)), - (rsa_private_key_pkcs8_import_stage52_key_a_vs_key_b, Some(0x7273615f69353032)), - (rsa_private_key_pkcs8_import_stage53_key_a_vs_key_b, Some(0x7273615f69353033)), - (rsa_private_key_pkcs8_import_stage54_key_a_vs_key_b, Some(0x7273615f69353034)), + ( + rsa_private_key_pkcs8_validate_stage0_key_a_vs_key_b, + Some(0x7273615f76733030) + ), + ( + rsa_private_key_pkcs8_validate_stage1_key_a_vs_key_b, + Some(0x7273615f76733031) + ), + ( + rsa_private_key_pkcs8_validate_stage2_key_a_vs_key_b, + Some(0x7273615f76733032) + ), + ( + rsa_private_key_pkcs8_validate_stage3_key_a_vs_key_b, + Some(0x7273615f76733033) + ), + ( + rsa_private_key_pkcs8_validate_stage4_key_a_vs_key_b, + Some(0x7273615f76733034) + ), + ( + rsa_private_key_pkcs8_validate_stage30_key_a_vs_key_b, + Some(0x7273615f76333030) + ), + ( + rsa_private_key_pkcs8_validate_stage31_key_a_vs_key_b, + Some(0x7273615f76333031) + ), + ( + rsa_private_key_pkcs8_validate_stage32_key_a_vs_key_b, + Some(0x7273615f76333032) + ), + ( + rsa_private_key_pkcs8_validate_stage40_key_a_vs_key_b, + Some(0x7273615f76343030) + ), + ( + rsa_private_key_pkcs8_validate_stage41_key_a_vs_key_b, + Some(0x7273615f76343031) + ), + ( + rsa_private_key_pkcs8_validate_stage42_key_a_vs_key_b, + Some(0x7273615f76343032) + ), + ( + rsa_private_key_pkcs8_import_stage50_key_a_vs_key_b, + Some(0x7273615f69353030) + ), + ( + rsa_private_key_pkcs8_import_stage51_key_a_vs_key_b, + Some(0x7273615f69353031) + ), + ( + rsa_private_key_pkcs8_import_stage52_key_a_vs_key_b, + Some(0x7273615f69353032) + ), + ( + rsa_private_key_pkcs8_import_stage53_key_a_vs_key_b, + Some(0x7273615f69353033) + ), + ( + rsa_private_key_pkcs8_import_stage54_key_a_vs_key_b, + Some(0x7273615f69353034) + ), (rsa_private_key_pkcs8_export_key_a_vs_key_b, Some(0x7273615f6b657978)), (hkdf_sha2_fixed_vs_random_ikm, Some(0x686b64665f736861)), (pbkdf2_sha2_fixed_vs_random_password, Some(0x70626b6466325f73)), diff --git a/tools/ct-harness/src/lib.rs b/tools/ct-harness/src/lib.rs index 8e3e8c4b..14acccae 100644 --- a/tools/ct-harness/src/lib.rs +++ b/tools/ct-harness/src/lib.rs @@ -6,7 +6,6 @@ // C ABI harness functions must remain plain `extern "C"` symbols. Pointer validity is documented // on the shared helpers and each entrypoint rejects null shapes before copying fixed-size inputs. -#![allow(clippy::not_unsafe_ptr_arg_deref)] use core::{ptr, slice}; use std::format; @@ -17,12 +16,11 @@ use rscrypto::{ Blake2bKey, Blake2s128, Blake2s256, Blake2sKey, Blake3, Blake3KeyedHash, ChaCha20Poly1305, ChaCha20Poly1305Key, Crc32, EcdsaP256SecretKey, EcdsaP384SecretKey, Ed25519PublicKey, Ed25519SecretKey, Ed25519Signature, HkdfSha256, HkdfSha384, HmacSha3_224Tag, HmacSha256, HmacSha256Tag, HmacSha384, HmacSha384Tag, HmacSha512, HmacSha512Tag, - Kmac256, MlKem512, - MlKem512Ciphertext, MlKem512DecapsulationKey, MlKem512EncapsulationKey, MlKem768, MlKem768Ciphertext, - MlKem768DecapsulationKey, MlKem768EncapsulationKey, MlKem1024, MlKem1024Ciphertext, MlKem1024DecapsulationKey, - MlKem1024EncapsulationKey, MlKemError, Pbkdf2Sha256, Pbkdf2Sha512, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, - RsaPssProfile, RsaPublicKeyPolicy, Scrypt, ScryptParams, SecretBytes, Sha256, X25519PublicKey, X25519SecretKey, - XChaCha20Poly1305, XChaCha20Poly1305Key, + Kmac256, MlKem512, MlKem512Ciphertext, MlKem512DecapsulationKey, MlKem512EncapsulationKey, MlKem768, + MlKem768Ciphertext, MlKem768DecapsulationKey, MlKem768EncapsulationKey, MlKem1024, MlKem1024Ciphertext, + MlKem1024DecapsulationKey, MlKem1024EncapsulationKey, MlKemError, Pbkdf2Sha256, Pbkdf2Sha512, RsaBlindingPair, + RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKeyPolicy, Scrypt, ScryptParams, + SecretBytes, Sha256, X25519PublicKey, X25519SecretKey, XChaCha20Poly1305, XChaCha20Poly1305Key, aead::{Nonce96, Nonce128, Nonce192, Nonce256}, checksum::Checksum, traits::Kem as _, @@ -122,13 +120,16 @@ unsafe fn read_mlkem_polyvec(ptr: *const u16) -> Option<[[u16; 2 } let mut out = [[0u16; 256]; K]; + let coefficient_count = K.strict_mul(256); // SAFETY: Copies a fixed-size ML-KEM polyvec from FFI memory because: - // 1. The caller contract requires `ptr` to be valid for reads of `K * 256` initialized `u16` - // values. - // 2. The caller contract requires `ptr` to be aligned for `u16`. - // 3. `out` is stack-owned storage valid for writes of `K * 256` `u16` values. - // 4. Source and destination cannot overlap because `out` is newly allocated stack storage. - unsafe { ptr::copy_nonoverlapping(ptr, out.as_mut_ptr().cast::(), K * 256) }; + // 1. `coefficient_count` is exactly `K * 256`; strict multiplication rejects an impossible + // destination layout instead of wrapping the copy length. + // 2. The caller contract requires `ptr` to be valid for reads of `coefficient_count` initialized + // `u16` values. + // 3. The caller contract requires `ptr` to be aligned for `u16`. + // 4. `out` is stack-owned storage valid for writes of `coefficient_count` `u16` values. + // 5. Source and destination cannot overlap because `out` is newly allocated stack storage. + unsafe { ptr::copy_nonoverlapping(ptr, out.as_mut_ptr().cast::(), coefficient_count) }; Some(out) } @@ -157,23 +158,36 @@ fn rsa_ct_fixture_key(pkcs8_der: &[u8]) -> Option { } /// HMAC-SHA256 tag verification harness. +/// +/// # Safety +/// +/// - When non-null, `key` and `data` must each reference a single allocation containing the +/// corresponding number of initialized bytes. Each byte length must be no greater than +/// `isize::MAX`, and neither range may be mutated during the call. +/// - When non-null, `expected_tag` must be valid for reads of 32 initialized bytes and must not be +/// mutated while copied. +/// - Null key and data pointers represent empty slices only when their corresponding lengths are +/// zero. Other null shapes are rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_hmac_sha256_verify( +pub unsafe extern "C" fn ct_entry_hmac_sha256_verify( key: *const u8, key_len: usize, data: *const u8, data_len: usize, expected_tag: *const u8, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice` / `read_array`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, immutability, + // and lifetime required to borrow a non-null `key`; `input_slice` handles both null shapes. let Some(key) = (unsafe { input_slice(key, key_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice` / `read_array`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, immutability, + // and lifetime required to borrow non-null `data`; `input_slice` handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; - // SAFETY: The expected tag pointer must reference exactly 32 readable bytes. + // SAFETY: The function contract requires a non-null `expected_tag` to reference exactly 32 + // readable, initialized bytes that remain immutable while copied into owned storage. let Some(expected_tag) = (unsafe { read_array::<32>(expected_tag) }) else { return STATUS_ERR; }; @@ -185,23 +199,36 @@ pub extern "C" fn ct_entry_hmac_sha256_verify( } /// HMAC-SHA256 64-bit truncated-tag verification harness. +/// +/// # Safety +/// +/// - When non-null, `key` and `data` must each reference a single allocation containing the +/// corresponding number of initialized bytes. Each byte length must be no greater than +/// `isize::MAX`, and neither range may be mutated during the call. +/// - When non-null, `expected_tag` must be valid for reads of 8 initialized bytes and must not be +/// mutated while copied. +/// - Null key and data pointers represent empty slices only when their corresponding lengths are +/// zero. Other null shapes are rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_hmac_sha256_verify_truncated_64( +pub unsafe extern "C" fn ct_entry_hmac_sha256_verify_truncated_64( key: *const u8, key_len: usize, data: *const u8, data_len: usize, expected_tag: *const u8, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice` / `read_array`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, immutability, + // and lifetime required to borrow a non-null `key`; `input_slice` handles both null shapes. let Some(key) = (unsafe { input_slice(key, key_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice` / `read_array`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, immutability, + // and lifetime required to borrow non-null `data`; `input_slice` handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; - // SAFETY: The expected tag pointer must reference exactly 8 readable bytes. + // SAFETY: The function contract requires a non-null `expected_tag` to reference exactly 8 + // readable, initialized bytes that remain immutable while copied into owned storage. let Some(expected_tag) = (unsafe { read_array::<8>(expected_tag) }) else { return STATUS_ERR; }; @@ -212,22 +239,35 @@ pub extern "C" fn ct_entry_hmac_sha256_verify_truncated_64( } /// BLAKE3 keyed-tag verification harness. +/// +/// # Safety +/// +/// - When non-null, `key` and `expected_tag` must each be valid for reads of 32 initialized bytes +/// and must not be mutated while copied. Null pointers are rejected. +/// - When non-null, `data` must reference a single allocation containing `data_len` initialized +/// bytes. `data_len` must be no greater than `isize::MAX`, and the range must not be mutated +/// during the call. +/// - A null `data` pointer represents an empty slice only when `data_len == 0`; other null shapes +/// are rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_blake3_verify_keyed( +pub unsafe extern "C" fn ct_entry_blake3_verify_keyed( key: *const u8, data: *const u8, data_len: usize, expected_tag: *const u8, ) -> u8 { - // SAFETY: The key pointer must reference exactly 32 readable bytes. + // SAFETY: The function contract requires a non-null `key` to reference exactly 32 readable, + // initialized bytes that remain immutable while copied into owned storage. let Some(key) = (unsafe { read_array::<32>(key) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, immutability, + // and lifetime required to borrow non-null `data`; `input_slice` handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; - // SAFETY: The expected tag pointer must reference exactly 32 readable bytes. + // SAFETY: The function contract requires a non-null `expected_tag` to reference exactly 32 + // readable, initialized bytes that remain immutable while copied into owned storage. let Some(expected_tag) = (unsafe { read_array::<32>(expected_tag) }) else { return STATUS_ERR; }; @@ -239,8 +279,23 @@ pub extern "C" fn ct_entry_blake3_verify_keyed( } /// ChaCha20-Poly1305 open/authentication harness. +/// +/// # Safety +/// +/// - When non-null, `key`, `nonce`, and `tag` must be valid for reads of 32, 12, and 16 initialized +/// bytes respectively and must not be mutated while copied. Null pointers are rejected. +/// - When non-null, `aad` must reference a single allocation containing `aad_len` initialized +/// bytes. `aad_len` must be no greater than `isize::MAX`, and the range must not be mutated +/// during the call. +/// - When non-null, `buffer` must reference a single allocation containing `buffer_len` initialized +/// bytes. `buffer_len` must be no greater than `isize::MAX`, and the range must remain exclusively +/// accessible for reads and writes during the call. +/// - Null `aad` and `buffer` pointers represent empty slices only when their corresponding lengths +/// are zero; other null shapes are rejected. +/// - The buffer range must not overlap the AAD or tag ranges. It may overlap the key or nonce ranges +/// because those inputs are copied before the mutable buffer borrow begins. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_chacha20poly1305_open( +pub unsafe extern "C" fn ct_entry_chacha20poly1305_open( key: *const u8, nonce: *const u8, aad: *const u8, @@ -249,23 +304,30 @@ pub extern "C" fn ct_entry_chacha20poly1305_open( buffer_len: usize, tag: *const u8, ) -> u8 { - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The function contract requires a non-null `key` to reference exactly 32 readable, + // initialized bytes that remain immutable while copied into owned storage. let Some(key) = (unsafe { read_array::<32>(key) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The function contract requires a non-null `nonce` to reference exactly 12 readable, + // initialized bytes that remain immutable while copied into owned storage. let Some(nonce) = (unsafe { read_array::<12>(nonce) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, immutability, + // and lifetime required to borrow non-null `aad`; `input_slice` handles both null shapes. let Some(aad) = (unsafe { input_slice(aad, aad_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI output pointer is validated by `output_slice`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, exclusivity, + // lifetime, and disjointness from the live AAD and tag ranges required to borrow `buffer` + // mutably; `output_slice` handles both null shapes. let Some(buffer) = (unsafe { output_slice(buffer, buffer_len) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires a non-null `tag` to reference exactly 16 readable, + // initialized bytes that remain immutable while copied and do not overlap the live mutable + // buffer borrow. let Some(tag) = (unsafe { read_array::<16>(tag) }) else { return STATUS_ERR; }; @@ -284,13 +346,23 @@ pub extern "C" fn ct_entry_chacha20poly1305_open( } /// X25519 scalar multiplication / shared-secret harness. +/// +/// # Safety +/// +/// - When non-null, `scalar` and `point` must each be valid for reads of 32 initialized bytes and +/// must not be mutated while copied. +/// - When non-null, `out` must be valid for writes of 32 bytes and must not be accessed while +/// written. +/// - Every non-null pointer must remain valid for the duration of the call. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_x25519(out: *mut u8, scalar: *const u8, point: *const u8) -> u8 { - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. +pub unsafe extern "C" fn ct_entry_x25519(out: *mut u8, scalar: *const u8, point: *const u8) -> u8 { + // SAFETY: The function contract requires a non-null `scalar` to reference exactly 32 readable, + // initialized bytes that remain valid while `read_array` copies them into owned storage. let Some(scalar) = (unsafe { read_array::<32>(scalar) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The function contract requires a non-null `point` to reference exactly 32 readable, + // initialized bytes that remain valid while `read_array` copies them into owned storage. let Some(point) = (unsafe { read_array::<32>(point) }) else { return STATUS_ERR; }; @@ -300,7 +372,8 @@ pub extern "C" fn ct_entry_x25519(out: *mut u8, scalar: *const u8, point: *const let Ok(shared) = secret.diffie_hellman(&public) else { return STATUS_ERR; }; - // SAFETY: The output pointer must reference exactly 32 writable bytes. + // SAFETY: The function contract requires a non-null `out` to reference exactly 32 writable bytes + // that remain valid and unaccessed while `write_array` copies the initialized shared secret. if unsafe { write_array(out, shared.as_bytes()) } { STATUS_OK } else { @@ -318,8 +391,24 @@ macro_rules! mlkem_ct_harness { $decapsulation_key:ty, $ciphertext:ty ) => { + /// Generate an ML-KEM key pair through the constant-time C ABI harness. + /// + /// # Safety + /// + /// - `encapsulation_key_out` must be valid for writes of + /// `<$profile>::ENCAPSULATION_KEY_SIZE` bytes. + /// - `decapsulation_key_out` must be valid for writes of + /// `<$profile>::DECAPSULATION_KEY_SIZE` bytes. + /// - `random` must be valid for reads of `<$profile>::KEY_GENERATION_RANDOM_SIZE` initialized + /// bytes. + /// - Each pointer must remain valid for the duration of the call, and writable regions must not + /// be accessed concurrently. #[unsafe(no_mangle)] - pub extern "C" fn $keygen(encapsulation_key_out: *mut u8, decapsulation_key_out: *mut u8, random: *const u8) -> u8 { + pub unsafe extern "C" fn $keygen( + encapsulation_key_out: *mut u8, + decapsulation_key_out: *mut u8, + random: *const u8, + ) -> u8 { // SAFETY: Copies ML-KEM key-generation randomness from FFI memory because: // 1. `read_array` rejects a null pointer before reading. // 2. The required length is exactly `<$profile>::KEY_GENERATION_RANDOM_SIZE`. @@ -353,18 +442,31 @@ macro_rules! mlkem_ct_harness { STATUS_OK } + /// Encapsulate an ML-KEM shared secret through the constant-time C ABI harness. + /// + /// # Safety + /// + /// - `ciphertext_out` must be valid for writes of `<$profile>::CIPHERTEXT_SIZE` bytes. + /// - `shared_secret_out` must be valid for writes of `<$profile>::SHARED_SECRET_SIZE` bytes. + /// - `encapsulation_key` must be valid for reads of `<$profile>::ENCAPSULATION_KEY_SIZE` + /// initialized bytes. + /// - `random` must be valid for reads of `<$profile>::ENCAPSULATION_RANDOM_SIZE` initialized + /// bytes. + /// - Each pointer must remain valid for the duration of the call, and writable regions must not + /// be accessed concurrently. #[unsafe(no_mangle)] - pub extern "C" fn $encapsulate( + pub unsafe extern "C" fn $encapsulate( ciphertext_out: *mut u8, shared_secret_out: *mut u8, encapsulation_key: *const u8, random: *const u8, ) -> u8 { - // SAFETY: Copies the ML-KEM encapsulation key from FFI memory because: - // 1. `read_array` rejects a null pointer before reading. - // 2. The required length is exactly `<$profile>::ENCAPSULATION_KEY_SIZE`. - // 3. The returned array is owned by this harness function before validation. let Some(encapsulation_key) = + // SAFETY: Copies the ML-KEM encapsulation key from FFI memory because: + // 1. `read_array` rejects a null pointer before reading. + // 2. The function contract requires exactly `<$profile>::ENCAPSULATION_KEY_SIZE` readable + // bytes. + // 3. The returned array is owned by this harness function before validation. (unsafe { read_array::<{ <$profile>::ENCAPSULATION_KEY_SIZE }>(encapsulation_key) }) else { return STATUS_ERR; @@ -405,17 +507,28 @@ macro_rules! mlkem_ct_harness { STATUS_OK } + /// Decapsulate an ML-KEM shared secret through the constant-time C ABI harness. + /// + /// # Safety + /// + /// - `shared_secret_out` must be valid for writes of `<$profile>::SHARED_SECRET_SIZE` bytes. + /// - `decapsulation_key` must be valid for reads of `<$profile>::DECAPSULATION_KEY_SIZE` + /// initialized bytes. + /// - `ciphertext` must be valid for reads of `<$profile>::CIPHERTEXT_SIZE` initialized bytes. + /// - Each pointer must remain valid for the duration of the call, and `shared_secret_out` must + /// not be accessed concurrently. #[unsafe(no_mangle)] - pub extern "C" fn $decapsulate( + pub unsafe extern "C" fn $decapsulate( shared_secret_out: *mut u8, decapsulation_key: *const u8, ciphertext: *const u8, ) -> u8 { - // SAFETY: Copies the ML-KEM decapsulation key from FFI memory because: - // 1. `read_array` rejects a null pointer before reading. - // 2. The required length is exactly `<$profile>::DECAPSULATION_KEY_SIZE`. - // 3. The returned array is owned by this harness function before use. let Some(decapsulation_key) = + // SAFETY: Copies the ML-KEM decapsulation key from FFI memory because: + // 1. `read_array` rejects a null pointer before reading. + // 2. The function contract requires exactly `<$profile>::DECAPSULATION_KEY_SIZE` readable + // bytes. + // 3. The returned array is owned by this harness function before use. (unsafe { read_array::<{ <$profile>::DECAPSULATION_KEY_SIZE }>(decapsulation_key) }) else { return STATUS_ERR; @@ -474,9 +587,16 @@ mlkem_ct_harness!( MlKem1024Ciphertext ); +/// Run the ML-KEM forward-NTT diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `poly` must be aligned for `u16`, valid for reads of 256 initialized +/// coefficients, and immutable while copied. A null pointer is rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_ntt(poly: *const u16) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. +pub unsafe extern "C" fn ct_entry_mlkem_diag_ntt(poly: *const u16) -> u16 { + // SAFETY: The function contract requires a non-null `poly` to be aligned for `u16` and valid for + // reads of exactly 256 initialized coefficients that remain immutable while copied. let Some(poly) = (unsafe { read_u16_array::<256>(poly) }) else { return 0; }; @@ -493,9 +613,16 @@ pub extern "C" fn ct_entry_mlkem_diag_ntt(poly: *const u16) -> u16 { rscrypto::auth::mlkem::diag_mlkem_ntt_input_digest(poly) } +/// Run the ML-KEM inverse-NTT diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `poly` must be aligned for `u16`, valid for reads of 256 initialized +/// coefficients, and immutable while copied. A null pointer is rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_inverse_ntt(poly: *const u16) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. +pub unsafe extern "C" fn ct_entry_mlkem_diag_inverse_ntt(poly: *const u16) -> u16 { + // SAFETY: The function contract requires a non-null `poly` to be aligned for `u16` and valid for + // reads of exactly 256 initialized coefficients that remain immutable while copied. let Some(poly) = (unsafe { read_u16_array::<256>(poly) }) else { return 0; }; @@ -512,9 +639,16 @@ pub extern "C" fn ct_entry_mlkem_diag_inverse_ntt(poly: *const u16) -> u16 { rscrypto::auth::mlkem::diag_mlkem_inverse_ntt_montgomery_product_input_digest(poly) } +/// Run the ML-KEM to-product-domain diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `poly` must be aligned for `u16`, valid for reads of 256 initialized +/// coefficients, and immutable while copied. A null pointer is rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_to_product_domain(poly: *const u16) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. +pub unsafe extern "C" fn ct_entry_mlkem_diag_to_product_domain(poly: *const u16) -> u16 { + // SAFETY: The function contract requires an aligned, initialized, immutable 256-coefficient + // source whenever poly is non-null; read_u16_array rejects null before copying. let Some(poly) = (unsafe { read_u16_array::<256>(poly) }) else { return 0; }; @@ -531,9 +665,16 @@ pub extern "C" fn ct_entry_mlkem_diag_to_product_domain(poly: *const u16) -> u16 rscrypto::auth::mlkem::diag_mlkem_to_montgomery_product_domain_input_digest(poly) } +/// Run the ML-KEM from-product-domain diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `poly` must be aligned for `u16`, valid for reads of 256 initialized +/// coefficients, and immutable while copied. A null pointer is rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_from_product_domain(poly: *const u16) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. +pub unsafe extern "C" fn ct_entry_mlkem_diag_from_product_domain(poly: *const u16) -> u16 { + // SAFETY: The function contract requires an aligned, initialized, immutable 256-coefficient + // source whenever poly is non-null; read_u16_array rejects null before copying. let Some(poly) = (unsafe { read_u16_array::<256>(poly) }) else { return 0; }; @@ -550,21 +691,31 @@ pub extern "C" fn ct_entry_mlkem_diag_from_product_domain(poly: *const u16) -> u rscrypto::auth::mlkem::diag_mlkem_from_montgomery_product_domain_input_digest(poly) } +/// Run the ML-KEM NTT multiply/add diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, each of `a`, `b`, and `acc` must be aligned for `u16`, valid for reads +/// of 256 initialized coefficients, and immutable while copied. Null pointers are rejected. +/// The three read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_multiply_ntts_add_assign( +pub unsafe extern "C" fn ct_entry_mlkem_diag_multiply_ntts_add_assign( a: *const u16, b: *const u16, acc: *const u16, ) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. + // SAFETY: The function contract requires an aligned, initialized, immutable 256-coefficient + // source whenever a is non-null; read_u16_array rejects null before copying. let Some(a) = (unsafe { read_u16_array::<256>(a) }) else { return 0; }; - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. + // SAFETY: The function contract provides the same guarantees for b. Its range may overlap a + // because both operations only read and each result is independently owned. let Some(b) = (unsafe { read_u16_array::<256>(b) }) else { return 0; }; - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. + // SAFETY: The function contract provides the same guarantees for acc. Its range may overlap a + // or b because all three operations only read and each result is independently owned. let Some(acc) = (unsafe { read_u16_array::<256>(acc) }) else { return 0; }; @@ -581,21 +732,31 @@ pub extern "C" fn ct_entry_mlkem_diag_multiply_ntts_add_assign( rscrypto::auth::mlkem::diag_mlkem_multiply_ntts_add_assign_input_digest(a, b, acc) } +/// Run the ML-KEM K=3 NTT accumulation diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `a` and `b` must each be aligned for `u16`, valid for reads of 768 +/// initialized coefficients, and immutable while copied. `acc` has the same obligations for +/// 256 coefficients. Null pointers are rejected. The three read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_multiply_ntts_accumulate_k3( +pub unsafe extern "C" fn ct_entry_mlkem_diag_multiply_ntts_accumulate_k3( a: *const u16, b: *const u16, acc: *const u16, ) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly three readable ML-KEM polynomials. + // SAFETY: The function contract requires an aligned, initialized, immutable 768-coefficient + // source whenever a is non-null; read_mlkem_polyvec rejects null before copying. let Some(a) = (unsafe { read_mlkem_polyvec::<3>(a) }) else { return 0; }; - // SAFETY: The diagnostic pointer must reference exactly three readable ML-KEM polynomials. + // SAFETY: The function contract provides the same guarantees for b. Its range may overlap a + // because both operations only read and each result is independently owned. let Some(b) = (unsafe { read_mlkem_polyvec::<3>(b) }) else { return 0; }; - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. + // SAFETY: The function contract requires an aligned, initialized, immutable 256-coefficient + // acc source. Its range may overlap a or b because all three operations only read. let Some(acc) = (unsafe { read_u16_array::<256>(acc) }) else { return 0; }; @@ -611,21 +772,31 @@ pub extern "C" fn ct_entry_mlkem_diag_multiply_ntts_accumulate_k3( rscrypto::auth::mlkem::diag_mlkem768_multiply_ntts_accumulate_input_digest(a, b, acc) } +/// Run the ML-KEM K=4 NTT accumulation diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `a` and `b` must each be aligned for `u16`, valid for reads of 1,024 +/// initialized coefficients, and immutable while copied. `acc` has the same obligations for +/// 256 coefficients. Null pointers are rejected. The three read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_multiply_ntts_accumulate_k4( +pub unsafe extern "C" fn ct_entry_mlkem_diag_multiply_ntts_accumulate_k4( a: *const u16, b: *const u16, acc: *const u16, ) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly four readable ML-KEM polynomials. + // SAFETY: The function contract requires an aligned, initialized, immutable 1,024-coefficient + // source whenever a is non-null; read_mlkem_polyvec rejects null before copying. let Some(a) = (unsafe { read_mlkem_polyvec::<4>(a) }) else { return 0; }; - // SAFETY: The diagnostic pointer must reference exactly four readable ML-KEM polynomials. + // SAFETY: The function contract provides the same guarantees for b. Its range may overlap a + // because both operations only read and each result is independently owned. let Some(b) = (unsafe { read_mlkem_polyvec::<4>(b) }) else { return 0; }; - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. + // SAFETY: The function contract requires an aligned, initialized, immutable 256-coefficient + // acc source. Its range may overlap a or b because all three operations only read. let Some(acc) = (unsafe { read_u16_array::<256>(acc) }) else { return 0; }; @@ -641,9 +812,16 @@ pub extern "C" fn ct_entry_mlkem_diag_multiply_ntts_accumulate_k4( rscrypto::auth::mlkem::diag_mlkem1024_multiply_ntts_accumulate_input_digest(a, b, acc) } +/// Run the ML-KEM compression/decompression diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `values` must be aligned for `u16`, valid for reads of four initialized +/// coefficients, and immutable while copied. A null pointer is rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_compress_decompress(values: *const u16) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly four readable ML-KEM coefficient values. +pub unsafe extern "C" fn ct_entry_mlkem_diag_compress_decompress(values: *const u16) -> u16 { + // SAFETY: The function contract requires an aligned, initialized, immutable four-coefficient + // source whenever values is non-null; read_u16_array rejects null before copying. let Some(values) = (unsafe { read_u16_array::<4>(values) }) else { return 0; }; @@ -660,26 +838,42 @@ pub extern "C" fn ct_entry_mlkem_diag_compress_decompress(values: *const u16) -> rscrypto::auth::mlkem::diag_mlkem_compress_decompress_values_digest(values) } -/// Ed25519 signing harness. +/// Sign a message with Ed25519 through the constant-time C ABI harness. +/// +/// # Safety +/// +/// - When non-null, `secret_key` must be valid for reads of 32 initialized bytes and immutable +/// while copied. A null pointer is rejected. +/// - A null `message` represents an empty slice only when `message_len` is zero. When non-null, +/// `message` must reference one allocation of `message_len` initialized bytes, `message_len` +/// must not exceed `isize::MAX`, and the range must remain immutable while signing reads it. +/// - When non-null, `out` must be valid for writes of 64 bytes and must not be accessed +/// concurrently while written. A null pointer is rejected. +/// - The output may overlap either input because both input reads finish before the output write. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_ed25519_sign( +pub unsafe extern "C" fn ct_entry_ed25519_sign( out: *mut u8, secret_key: *const u8, message: *const u8, message_len: usize, ) -> u8 { - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires a non-null secret_key to expose 32 initialized, + // readable bytes that remain immutable while read_array copies them into owned storage. let Some(secret_key) = (unsafe { read_array::<32>(secret_key) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointer is validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null message; input_slice handles both null + // shapes. let Some(message) = (unsafe { input_slice(message, message_len) }) else { return STATUS_ERR; }; let secret_key = Ed25519SecretKey::from_bytes(secret_key); let signature = secret_key.sign(message); - // SAFETY: The output pointer must reference exactly 64 writable bytes. + // SAFETY: The function contract requires a non-null out to expose 64 writable bytes without + // concurrent access. Both caller-memory reads have finished, so out may overlap either input; + // write_array rejects null before writing from the independently owned signature. if unsafe { write_array(out, signature.as_bytes()) } { STATUS_OK } else { @@ -687,24 +881,40 @@ pub extern "C" fn ct_entry_ed25519_sign( } } -/// ECDSA/P-256 signing harness with caller-supplied projective blinding. +/// Sign a message with ECDSA/P-256 and caller-supplied projective blinding. +/// +/// # Safety +/// +/// - When non-null, `secret_key` and `blind` must be valid for reads of 32 and 64 initialized +/// bytes, respectively, and immutable while copied. Null pointers are rejected. +/// - A null `message` represents an empty slice only when `message_len` is zero. When non-null, +/// `message` must reference one allocation of `message_len` initialized bytes, `message_len` +/// must not exceed `isize::MAX`, and the range must remain immutable while signing reads it. +/// - When non-null, `out` must be valid for writes of 64 bytes and must not be accessed +/// concurrently while written. A null pointer is rejected. +/// - The read-only inputs may overlap. The output may overlap any input because all input reads +/// finish before the output write. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_ecdsa_p256_sign( +pub unsafe extern "C" fn ct_entry_ecdsa_p256_sign( out: *mut u8, secret_key: *const u8, blind: *const u8, message: *const u8, message_len: usize, ) -> u8 { - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires a non-null secret_key to expose 32 initialized, + // readable bytes that remain immutable while read_array copies them into owned storage. let Some(secret_key) = (unsafe { read_array::<32>(secret_key) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires a non-null blind to expose 64 initialized, readable + // bytes that remain immutable while read_array copies them into owned storage. let Some(blind) = (unsafe { read_array::<64>(blind) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointer is validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null message; input_slice handles both null + // shapes. let Some(message) = (unsafe { input_slice(message, message_len) }) else { return STATUS_ERR; }; @@ -717,7 +927,9 @@ pub extern "C" fn ct_entry_ecdsa_p256_sign( }; let signature = signature.to_bytes(); - // SAFETY: The output pointer must reference exactly 64 writable bytes. + // SAFETY: The function contract requires a non-null out to expose 64 writable bytes without + // concurrent access. All caller-memory reads have finished, so out may overlap any input; + // write_array rejects null before writing from the independently owned signature. if unsafe { write_array(out, &signature) } { STATUS_OK } else { @@ -725,24 +937,40 @@ pub extern "C" fn ct_entry_ecdsa_p256_sign( } } -/// ECDSA/P-384 signing harness with caller-supplied projective blinding. +/// Sign a message with ECDSA/P-384 and caller-supplied projective blinding. +/// +/// # Safety +/// +/// - When non-null, `secret_key` and `blind` must be valid for reads of 48 and 96 initialized +/// bytes, respectively, and immutable while copied. Null pointers are rejected. +/// - A null `message` represents an empty slice only when `message_len` is zero. When non-null, +/// `message` must reference one allocation of `message_len` initialized bytes, `message_len` +/// must not exceed `isize::MAX`, and the range must remain immutable while signing reads it. +/// - When non-null, `out` must be valid for writes of 96 bytes and must not be accessed +/// concurrently while written. A null pointer is rejected. +/// - The read-only inputs may overlap. The output may overlap any input because all input reads +/// finish before the output write. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_ecdsa_p384_sign( +pub unsafe extern "C" fn ct_entry_ecdsa_p384_sign( out: *mut u8, secret_key: *const u8, blind: *const u8, message: *const u8, message_len: usize, ) -> u8 { - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires a non-null secret_key to expose 48 initialized, + // readable bytes that remain immutable while read_array copies them into owned storage. let Some(secret_key) = (unsafe { read_array::<48>(secret_key) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires a non-null blind to expose 96 initialized, readable + // bytes that remain immutable while read_array copies them into owned storage. let Some(blind) = (unsafe { read_array::<96>(blind) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointer is validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null message; input_slice handles both null + // shapes. let Some(message) = (unsafe { input_slice(message, message_len) }) else { return STATUS_ERR; }; @@ -755,7 +983,9 @@ pub extern "C" fn ct_entry_ecdsa_p384_sign( }; let signature = signature.to_bytes(); - // SAFETY: The output pointer must reference exactly 96 writable bytes. + // SAFETY: The function contract requires a non-null out to expose 96 writable bytes without + // concurrent access. All caller-memory reads have finished, so out may overlap any input; + // write_array rejects null before writing from the independently owned signature. if unsafe { write_array(out, &signature) } { STATUS_OK } else { @@ -763,9 +993,21 @@ pub extern "C" fn ct_entry_ecdsa_p384_sign( } } -/// PBKDF2-HMAC-SHA256 verification harness. +/// Verify a PBKDF2-HMAC-SHA256 password through the constant-time C ABI harness. +/// +/// # Safety +/// +/// Each `(pointer, length)` pair for `password`, `salt`, and `expected` must satisfy all of the +/// following: +/// +/// - a null pointer is used only with a zero length; +/// - a non-null pointer references one allocation of `length` initialized bytes; +/// - `length` does not exceed `isize::MAX`; and +/// - the referenced bytes remain immutable for the call. +/// +/// The three read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_pbkdf2_sha256_verify( +pub unsafe extern "C" fn ct_entry_pbkdf2_sha256_verify( password: *const u8, password_len: usize, salt: *const u8, @@ -774,15 +1016,21 @@ pub extern "C" fn ct_entry_pbkdf2_sha256_verify( expected: *const u8, expected_len: usize, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null password; input_slice handles both null + // shapes. let Some(password) = (unsafe { input_slice(password, password_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null salt; input_slice handles both null + // shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null expected value; input_slice handles both + // null shapes. let Some(expected) = (unsafe { input_slice(expected, expected_len) }) else { return STATUS_ERR; }; @@ -794,23 +1042,38 @@ pub extern "C" fn ct_entry_pbkdf2_sha256_verify( macro_rules! fixed_tag_verify_entry { ($name:ident, $ty:ty, $tag_ty:ty, $tag_len:literal) => { + #[doc = "Verify a fixed-size authentication tag through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "- Null `key` and `data` pointers are permitted only with matching zero lengths. Each"] + #[doc = " non-null pointer must reference one allocation of initialized bytes, its length must"] + #[doc = " not exceed `isize::MAX`, and the range must remain immutable for the call."] + #[doc = "- When non-null, `expected_tag` must be valid for reads of the fixed tag length,"] + #[doc = " contain initialized bytes, and remain immutable while copied. Null is rejected."] + #[doc = "- The three read-only ranges may overlap."] #[unsafe(no_mangle)] - pub extern "C" fn $name( + pub unsafe extern "C" fn $name( key: *const u8, key_len: usize, data: *const u8, data_len: usize, expected_tag: *const u8, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice` / `read_array`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for a non-null key; input_slice + // handles both null shapes. let Some(key) = (unsafe { input_slice(key, key_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice` / `read_array`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null data; input_slice + // handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; - // SAFETY: The expected tag pointer must reference exactly `$tag_len` readable bytes. + // SAFETY: The generated function contract requires a non-null expected_tag to expose + // $tag_len initialized, readable bytes that remain immutable while read_array copies them. let Some(expected_tag) = (unsafe { read_array::<$tag_len>(expected_tag) }) else { return STATUS_ERR; }; @@ -828,14 +1091,23 @@ fixed_tag_verify_entry!(ct_entry_hmac_sha512_verify, HmacSha512, HmacSha512Tag, macro_rules! fixed_owner_eq_entry { ($name:ident, $type:ty, $len:expr) => { + #[doc = "Compare two fixed-size secret-owning values through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "When non-null, `a` and `b` must each be valid for reads of the selected fixed length,"] + #[doc = "contain initialized bytes, and remain immutable while copied. Null is rejected. The"] + #[doc = "two read-only ranges may overlap."] #[inline(never)] #[unsafe(no_mangle)] - pub extern "C" fn $name(a: *const u8, b: *const u8) -> u8 { - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + pub unsafe extern "C" fn $name(a: *const u8, b: *const u8) -> u8 { + // SAFETY: The generated function contract requires non-null a to expose $len initialized, + // readable bytes that remain immutable while read_array copies them into owned storage. let Some(a) = (unsafe { read_array::<$len>(a) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The generated function contract requires non-null b to expose $len initialized, + // readable bytes that remain immutable while read_array copies them into owned storage. let Some(b) = (unsafe { read_array::<$len>(b) }) else { return STATUS_ERR; }; @@ -854,9 +1126,16 @@ fixed_owner_eq_entry!(ct_entry_owner_eq_1632, MlKem512DecapsulationKey, 1632); fixed_owner_eq_entry!(ct_entry_owner_eq_2400, MlKem768DecapsulationKey, 2400); fixed_owner_eq_entry!(ct_entry_owner_eq_3168, MlKem1024DecapsulationKey, 3168); +/// Check that a 32-byte secret's `Debug` representation remains redacted. +/// +/// # Safety +/// +/// When non-null, `secret` must be valid for reads of 32 initialized bytes and immutable while +/// copied. A null pointer is rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_secret_bytes32_debug_masked(secret: *const u8) -> u8 { - // SAFETY: Fixed-size FFI input is copied by value after null check. +pub unsafe extern "C" fn ct_entry_secret_bytes32_debug_masked(secret: *const u8) -> u8 { + // SAFETY: The function contract requires a non-null secret to expose 32 initialized, readable + // bytes that remain immutable while read_array copies them into owned storage. let Some(secret) = (unsafe { read_array::<32>(secret) }) else { return STATUS_ERR; }; @@ -865,8 +1144,21 @@ pub extern "C" fn ct_entry_secret_bytes32_debug_masked(secret: *const u8) -> u8 u8::from(formatted == "SecretBytes(****)") } +/// Verify a KMAC256 tag through the constant-time C ABI harness. +/// +/// # Safety +/// +/// Each `(pointer, length)` pair for `key`, `customization`, `data`, and `expected_tag` must +/// satisfy all of the following: +/// +/// - a null pointer is used only with a zero length; +/// - a non-null pointer references one allocation of `length` initialized bytes; +/// - `length` does not exceed `isize::MAX`; and +/// - the referenced bytes remain immutable for the call. +/// +/// The four read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_kmac256_verify( +pub unsafe extern "C" fn ct_entry_kmac256_verify( key: *const u8, key_len: usize, customization: *const u8, @@ -876,19 +1168,25 @@ pub extern "C" fn ct_entry_kmac256_verify( expected_tag: *const u8, expected_tag_len: usize, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null key; input_slice handles both null shapes. let Some(key) = (unsafe { input_slice(key, key_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null customization; input_slice handles both + // null shapes. let Some(customization) = (unsafe { input_slice(customization, customization_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null data; input_slice handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null expected_tag; input_slice handles both + // null shapes. let Some(expected_tag) = (unsafe { input_slice(expected_tag, expected_tag_len) }) else { return STATUS_ERR; }; @@ -900,8 +1198,26 @@ pub extern "C" fn ct_entry_kmac256_verify( macro_rules! aead_open_entry { ($name:ident, $cipher:ty, $key:ty, $nonce:ty, $key_len:literal, $nonce_len:literal) => { + #[doc = "Authenticate and decrypt a buffer in place through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "- Non-null `key` and `nonce` pointers must expose the selected fixed lengths of"] + #[doc = " initialized readable bytes and remain immutable while copied. Null is rejected."] + #[doc = "- A null `aad` pointer is permitted only when `aad_len` is zero. A non-null pointer"] + #[doc = " must reference one allocation of `aad_len` initialized bytes, `aad_len` must not"] + #[doc = " exceed `isize::MAX`, and the range must remain immutable for the call."] + #[doc = "- A null `buffer` pointer is permitted only when `buffer_len` is zero. A non-null"] + #[doc = " pointer must reference one allocation of `buffer_len` initialized writable bytes,"] + #[doc = " `buffer_len` must not exceed `isize::MAX`, and the range must be exclusively"] + #[doc = " accessed for the call."] + #[doc = "- When non-null, `tag` must expose 16 initialized readable bytes and remain immutable"] + #[doc = " while copied. Null is rejected."] + #[doc = "- Nonempty `buffer` ranges must be disjoint from `aad` and `tag`. The buffer may"] + #[doc = " overlap `key` or `nonce` because both are copied before its mutable borrow begins."] + #[doc = " Read-only ranges may overlap one another."] #[unsafe(no_mangle)] - pub extern "C" fn $name( + pub unsafe extern "C" fn $name( key: *const u8, nonce: *const u8, aad: *const u8, @@ -910,23 +1226,30 @@ macro_rules! aead_open_entry { buffer_len: usize, tag: *const u8, ) -> u8 { - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The generated function contract requires non-null key to expose $key_len + // initialized, readable bytes that remain immutable while read_array copies them. let Some(key) = (unsafe { read_array::<$key_len>(key) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The generated function contract requires non-null nonce to expose $nonce_len + // initialized, readable bytes that remain immutable while read_array copies them. let Some(nonce) = (unsafe { read_array::<$nonce_len>(nonce) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointer is validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null aad; input_slice handles + // both null shapes. let Some(aad) = (unsafe { input_slice(aad, aad_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI output pointer is validated by `output_slice`. + // SAFETY: The generated function contract establishes buffer's allocation, length bound, + // initialization, exclusivity, and lifetime. It also requires disjointness from the live aad + // and tag reads; output_slice handles both null shapes. let Some(buffer) = (unsafe { output_slice(buffer, buffer_len) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The generated function contract requires non-null tag to expose 16 initialized, + // readable bytes, remain immutable while copied, and be disjoint from the live buffer. let Some(tag) = (unsafe { read_array::<16>(tag) }) else { return STATUS_ERR; }; @@ -984,8 +1307,22 @@ aead_open_entry!( macro_rules! hkdf_derive_entry { ($name:ident, $ty:ty) => { + #[doc = "Derive HKDF output through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "- Null `salt`, `ikm`, and `info` pointers are permitted only with matching zero"] + #[doc = " lengths. Each non-null pointer must reference one allocation of initialized bytes,"] + #[doc = " its length must not exceed `isize::MAX`, and the range must remain immutable for"] + #[doc = " the call."] + #[doc = "- A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer"] + #[doc = " must reference one allocation of `out_len` initialized writable bytes, `out_len`"] + #[doc = " must not exceed `isize::MAX`, and the range must be exclusively accessed for the"] + #[doc = " call."] + #[doc = "- Input ranges may overlap one another. A nonempty output range must be disjoint from"] + #[doc = " every input range because the shared and mutable borrows remain live together."] #[unsafe(no_mangle)] - pub extern "C" fn $name( + pub unsafe extern "C" fn $name( salt: *const u8, salt_len: usize, ikm: *const u8, @@ -995,19 +1332,27 @@ macro_rules! hkdf_derive_entry { out: *mut u8, out_len: usize, ) -> u8 { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null salt; input_slice handles + // both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null ikm; input_slice handles + // both null shapes. let Some(ikm) = (unsafe { input_slice(ikm, ikm_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null info; input_slice handles + // both null shapes. let Some(info) = (unsafe { input_slice(info, info_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes out's allocation, length bound, + // initialization, exclusivity, and lifetime, including disjointness from all live inputs; + // output_slice handles both null shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return STATUS_ERR; }; @@ -1024,8 +1369,22 @@ hkdf_derive_entry!(ct_entry_hkdf_sha384_derive, HkdfSha384); macro_rules! pbkdf2_entry { ($derive_name:ident, $verify_name:ident, $ty:ty) => { + #[doc = "Derive PBKDF2 output through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "- Null `password` and `salt` pointers are permitted only with matching zero lengths."] + #[doc = " Each non-null pointer must reference one allocation of initialized bytes, its"] + #[doc = " length must not exceed `isize::MAX`, and the range must remain immutable for the"] + #[doc = " call."] + #[doc = "- A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer"] + #[doc = " must reference one allocation of `out_len` initialized writable bytes, `out_len`"] + #[doc = " must not exceed `isize::MAX`, and the range must be exclusively accessed for the"] + #[doc = " call."] + #[doc = "- Password and salt may overlap. A nonempty output range must be disjoint from both"] + #[doc = " input ranges because their shared and mutable borrows remain live together."] #[unsafe(no_mangle)] - pub extern "C" fn $derive_name( + pub unsafe extern "C" fn $derive_name( password: *const u8, password_len: usize, salt: *const u8, @@ -1034,15 +1393,21 @@ macro_rules! pbkdf2_entry { out: *mut u8, out_len: usize, ) -> u8 { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null password; input_slice + // handles both null shapes. let Some(password) = (unsafe { input_slice(password, password_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null salt; input_slice handles + // both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes out's allocation, length bound, + // initialization, exclusivity, and lifetime, including disjointness from both live inputs; + // output_slice handles both null shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return STATUS_ERR; }; @@ -1052,8 +1417,16 @@ macro_rules! pbkdf2_entry { .unwrap_or(STATUS_ERR) } + #[doc = "Verify PBKDF2 output through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "Null `password`, `salt`, and `expected` pointers are permitted only with matching"] + #[doc = "zero lengths. Each non-null pointer must reference one allocation of initialized"] + #[doc = "bytes, its length must not exceed `isize::MAX`, and the range must remain immutable"] + #[doc = "for the call. The three read-only ranges may overlap."] #[unsafe(no_mangle)] - pub extern "C" fn $verify_name( + pub unsafe extern "C" fn $verify_name( password: *const u8, password_len: usize, salt: *const u8, @@ -1062,15 +1435,21 @@ macro_rules! pbkdf2_entry { expected: *const u8, expected_len: usize, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null password; input_slice + // handles both null shapes. let Some(password) = (unsafe { input_slice(password, password_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null salt; input_slice handles + // both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null expected output; + // input_slice handles both null shapes. let Some(expected) = (unsafe { input_slice(expected, expected_len) }) else { return STATUS_ERR; }; @@ -1095,8 +1474,22 @@ pbkdf2_entry!( macro_rules! argon2_entry { ($derive_name:ident, $verify_name:ident, $ty:ty) => { + #[doc = "Derive Argon2 output through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "- Null `password` and `salt` pointers are permitted only with matching zero lengths."] + #[doc = " Each non-null pointer must reference one allocation of initialized bytes, its"] + #[doc = " length must not exceed `isize::MAX`, and the range must remain immutable for the"] + #[doc = " call."] + #[doc = "- A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer"] + #[doc = " must reference one allocation of `out_len` initialized writable bytes, `out_len`"] + #[doc = " must not exceed `isize::MAX`, and the range must be exclusively accessed for the"] + #[doc = " call."] + #[doc = "- Password and salt may overlap. A nonempty output range must be disjoint from both"] + #[doc = " input ranges because their shared and mutable borrows remain live together."] #[unsafe(no_mangle)] - pub extern "C" fn $derive_name( + pub unsafe extern "C" fn $derive_name( password: *const u8, password_len: usize, salt: *const u8, @@ -1106,15 +1499,21 @@ macro_rules! argon2_entry { out: *mut u8, out_len: usize, ) -> u8 { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null password; input_slice + // handles both null shapes. let Some(password) = (unsafe { input_slice(password, password_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null salt; input_slice handles + // both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes out's allocation, length bound, + // initialization, exclusivity, and lifetime, including disjointness from both live inputs; + // output_slice handles both null shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return STATUS_ERR; }; @@ -1127,8 +1526,16 @@ macro_rules! argon2_entry { .unwrap_or(STATUS_ERR) } + #[doc = "Verify Argon2 output through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "Null `password`, `salt`, and `expected` pointers are permitted only with matching"] + #[doc = "zero lengths. Each non-null pointer must reference one allocation of initialized"] + #[doc = "bytes, its length must not exceed `isize::MAX`, and the range must remain immutable"] + #[doc = "for the call. The three read-only ranges may overlap."] #[unsafe(no_mangle)] - pub extern "C" fn $verify_name( + pub unsafe extern "C" fn $verify_name( password: *const u8, password_len: usize, salt: *const u8, @@ -1138,15 +1545,21 @@ macro_rules! argon2_entry { expected: *const u8, expected_len: usize, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null password; input_slice + // handles both null shapes. let Some(password) = (unsafe { input_slice(password, password_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null salt; input_slice handles + // both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null expected output; + // input_slice handles both null shapes. let Some(expected) = (unsafe { input_slice(expected, expected_len) }) else { return STATUS_ERR; }; @@ -1165,8 +1578,21 @@ argon2_entry!(ct_entry_argon2i_hash, ct_entry_argon2i_verify, Argon2i); argon2_entry!(ct_entry_argon2d_hash, ct_entry_argon2d_verify, Argon2d); argon2_entry!(ct_entry_argon2id_hash, ct_entry_argon2id_verify, Argon2id); +/// Verify scrypt output through the constant-time C ABI harness. +/// +/// # Safety +/// +/// Each `(pointer, length)` pair for `password`, `salt`, and `expected` must satisfy all of the +/// following: +/// +/// - a null pointer is used only with a zero length; +/// - a non-null pointer references one allocation of `length` initialized bytes; +/// - `length` does not exceed `isize::MAX`; and +/// - the referenced bytes remain immutable for the call. +/// +/// The three read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_scrypt_verify( +pub unsafe extern "C" fn ct_entry_scrypt_verify( password: *const u8, password_len: usize, salt: *const u8, @@ -1177,20 +1603,24 @@ pub extern "C" fn ct_entry_scrypt_verify( expected: *const u8, expected_len: usize, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null password; input_slice handles both null + // shapes. let Some(password) = (unsafe { input_slice(password, password_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null salt; input_slice handles both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null expected output; input_slice handles both + // null shapes. let Some(expected) = (unsafe { input_slice(expected, expected_len) }) else { return STATUS_ERR; }; - let Ok(params) = ScryptParams::new(log_n, r, p) - else { + let Ok(params) = ScryptParams::new(log_n, r, p) else { return STATUS_ERR; }; @@ -1199,22 +1629,36 @@ pub extern "C" fn ct_entry_scrypt_verify( .unwrap_or(STATUS_ERR) } +/// Verify an Ed25519 signature through the constant-time C ABI harness. +/// +/// # Safety +/// +/// - When non-null, `public_key` and `signature` must be valid for reads of 32 and 64 initialized +/// bytes, respectively, and immutable while copied. Null pointers are rejected. +/// - A null `message` pointer is permitted only when `message_len` is zero. A non-null pointer +/// must reference one allocation of `message_len` initialized bytes, `message_len` must not +/// exceed `isize::MAX`, and the range must remain immutable for the call. +/// - The three read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_ed25519_verify( +pub unsafe extern "C" fn ct_entry_ed25519_verify( public_key: *const u8, message: *const u8, message_len: usize, signature: *const u8, ) -> u8 { - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The function contract requires non-null public_key to expose 32 initialized, readable + // bytes that remain immutable while read_array copies them into owned storage. let Some(public_key) = (unsafe { read_array::<32>(public_key) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointer is validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null message; input_slice handles both null + // shapes. let Some(message) = (unsafe { input_slice(message, message_len) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires non-null signature to expose 64 initialized, readable + // bytes that remain immutable while read_array copies them into owned storage. let Some(signature) = (unsafe { read_array::<64>(signature) }) else { return STATUS_ERR; }; @@ -1225,8 +1669,21 @@ pub extern "C" fn ct_entry_ed25519_verify( .unwrap_or(STATUS_ERR) } +/// Sign with RSA PKCS#1 v1.5 and caller-supplied blinding through the CT harness. +/// +/// # Safety +/// +/// - A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer must +/// reference one allocation of `out_len` initialized writable bytes, `out_len` must not exceed +/// `isize::MAX`, and the range must be exclusively accessed for the call. +/// - Each `(pointer, length)` pair for `pkcs8_der`, `message`, `blinding_factor`, and +/// `blinding_inverse` may use a null pointer only with a zero length. Every non-null pointer must +/// reference one allocation of initialized bytes, its length must not exceed `isize::MAX`, and +/// the range must remain immutable for the call. +/// - A nonempty output range must be disjoint from every input range. Read-only input ranges may +/// overlap one another. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_rsa_pkcs1v15_sign_fixed_blinding( +pub unsafe extern "C" fn ct_entry_rsa_pkcs1v15_sign_fixed_blinding( out: *mut u8, out_len: usize, pkcs8_der: *const u8, @@ -1238,23 +1695,33 @@ pub extern "C" fn ct_entry_rsa_pkcs1v15_sign_fixed_blinding( blinding_inverse: *const u8, blinding_inverse_len: usize, ) -> u8 { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes out's allocation, length bound, initialization, + // exclusivity, lifetime, and disjointness from every live input; output_slice handles both null + // shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null pkcs8_der; input_slice + // handles both null shapes. let Some(pkcs8_der) = (unsafe { input_slice(pkcs8_der, pkcs8_der_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null message; input_slice + // handles both null shapes. let Some(message) = (unsafe { input_slice(message, message_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_factor; + // input_slice handles both null shapes. let Some(blinding_factor) = (unsafe { input_slice(blinding_factor, blinding_factor_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_inverse; + // input_slice handles both null shapes. let Some(blinding_inverse) = (unsafe { input_slice(blinding_inverse, blinding_inverse_len) }) else { return STATUS_ERR; }; @@ -1267,8 +1734,7 @@ pub extern "C" fn ct_entry_rsa_pkcs1v15_sign_fixed_blinding( .sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, message, - blinding_factor, - blinding_inverse, + RsaBlindingPair::new(blinding_factor, blinding_inverse), out, &mut scratch, ) @@ -1276,8 +1742,21 @@ pub extern "C" fn ct_entry_rsa_pkcs1v15_sign_fixed_blinding( .unwrap_or(STATUS_ERR) } +/// Sign with RSA-PSS and caller-supplied salt and blinding through the CT harness. +/// +/// # Safety +/// +/// - A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer must +/// reference one allocation of `out_len` initialized writable bytes, `out_len` must not exceed +/// `isize::MAX`, and the range must be exclusively accessed for the call. +/// - Each `(pointer, length)` pair for `pkcs8_der`, `message`, `salt`, `blinding_factor`, and +/// `blinding_inverse` may use a null pointer only with a zero length. Every non-null pointer must +/// reference one allocation of initialized bytes, its length must not exceed `isize::MAX`, and +/// the range must remain immutable for the call. +/// - A nonempty output range must be disjoint from every input range. Read-only input ranges may +/// overlap one another. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_rsa_pss_sign_fixed_blinding( +pub unsafe extern "C" fn ct_entry_rsa_pss_sign_fixed_blinding( out: *mut u8, out_len: usize, pkcs8_der: *const u8, @@ -1291,27 +1770,39 @@ pub extern "C" fn ct_entry_rsa_pss_sign_fixed_blinding( blinding_inverse: *const u8, blinding_inverse_len: usize, ) -> u8 { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes out's allocation, length bound, initialization, + // exclusivity, lifetime, and disjointness from every live input; output_slice handles both null + // shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null pkcs8_der; input_slice + // handles both null shapes. let Some(pkcs8_der) = (unsafe { input_slice(pkcs8_der, pkcs8_der_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null message; input_slice + // handles both null shapes. let Some(message) = (unsafe { input_slice(message, message_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null salt; input_slice + // handles both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_factor; + // input_slice handles both null shapes. let Some(blinding_factor) = (unsafe { input_slice(blinding_factor, blinding_factor_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_inverse; + // input_slice handles both null shapes. let Some(blinding_inverse) = (unsafe { input_slice(blinding_inverse, blinding_inverse_len) }) else { return STATUS_ERR; }; @@ -1325,8 +1816,7 @@ pub extern "C" fn ct_entry_rsa_pss_sign_fixed_blinding( RsaPssProfile::Sha256, message, salt, - blinding_factor, - blinding_inverse, + RsaBlindingPair::new(blinding_factor, blinding_inverse), out, &mut scratch, ) @@ -1334,8 +1824,21 @@ pub extern "C" fn ct_entry_rsa_pss_sign_fixed_blinding( .unwrap_or(STATUS_ERR) } +/// Decrypt RSA-OAEP with caller-supplied blinding through the CT harness. +/// +/// # Safety +/// +/// - A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer must +/// reference one allocation of `out_len` initialized writable bytes, `out_len` must not exceed +/// `isize::MAX`, and the range must be exclusively accessed for the call. +/// - Each `(pointer, length)` pair for `pkcs8_der`, `label`, `ciphertext`, `blinding_factor`, and +/// `blinding_inverse` may use a null pointer only with a zero length. Every non-null pointer must +/// reference one allocation of initialized bytes, its length must not exceed `isize::MAX`, and +/// the range must remain immutable for the call. +/// - A nonempty output range must be disjoint from every input range. Read-only input ranges may +/// overlap one another. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_rsa_oaep_decrypt_fixed_blinding( +pub unsafe extern "C" fn ct_entry_rsa_oaep_decrypt_fixed_blinding( out: *mut u8, out_len: usize, pkcs8_der: *const u8, @@ -1349,27 +1852,39 @@ pub extern "C" fn ct_entry_rsa_oaep_decrypt_fixed_blinding( blinding_inverse: *const u8, blinding_inverse_len: usize, ) -> usize { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes out's allocation, length bound, initialization, + // exclusivity, lifetime, and disjointness from every live input; output_slice handles both null + // shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null pkcs8_der; input_slice + // handles both null shapes. let Some(pkcs8_der) = (unsafe { input_slice(pkcs8_der, pkcs8_der_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null label; input_slice + // handles both null shapes. let Some(label) = (unsafe { input_slice(label, label_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null ciphertext; input_slice + // handles both null shapes. let Some(ciphertext) = (unsafe { input_slice(ciphertext, ciphertext_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_factor; + // input_slice handles both null shapes. let Some(blinding_factor) = (unsafe { input_slice(blinding_factor, blinding_factor_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_inverse; + // input_slice handles both null shapes. let Some(blinding_inverse) = (unsafe { input_slice(blinding_inverse, blinding_inverse_len) }) else { return usize::MAX; }; @@ -1383,16 +1898,28 @@ pub extern "C" fn ct_entry_rsa_oaep_decrypt_fixed_blinding( RsaOaepProfile::Sha256, label, ciphertext, - blinding_factor, - blinding_inverse, + RsaBlindingPair::new(blinding_factor, blinding_inverse), out, &mut scratch, ) .unwrap_or(usize::MAX) } +/// Decrypt RSAES-PKCS1-v1_5 with caller-supplied blinding through the CT harness. +/// +/// # Safety +/// +/// - A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer must +/// reference one allocation of `out_len` initialized writable bytes, `out_len` must not exceed +/// `isize::MAX`, and the range must be exclusively accessed for the call. +/// - Each `(pointer, length)` pair for `pkcs8_der`, `ciphertext`, `blinding_factor`, and +/// `blinding_inverse` may use a null pointer only with a zero length. Every non-null pointer must +/// reference one allocation of initialized bytes, its length must not exceed `isize::MAX`, and +/// the range must remain immutable for the call. +/// - A nonempty output range must be disjoint from every input range. Read-only input ranges may +/// overlap one another. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding( +pub unsafe extern "C" fn ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding( out: *mut u8, out_len: usize, pkcs8_der: *const u8, @@ -1404,23 +1931,33 @@ pub extern "C" fn ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding( blinding_inverse: *const u8, blinding_inverse_len: usize, ) -> usize { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes out's allocation, length bound, initialization, + // exclusivity, lifetime, and disjointness from every live input; output_slice handles both null + // shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null pkcs8_der; input_slice + // handles both null shapes. let Some(pkcs8_der) = (unsafe { input_slice(pkcs8_der, pkcs8_der_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null ciphertext; input_slice + // handles both null shapes. let Some(ciphertext) = (unsafe { input_slice(ciphertext, ciphertext_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_factor; + // input_slice handles both null shapes. let Some(blinding_factor) = (unsafe { input_slice(blinding_factor, blinding_factor_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_inverse; + // input_slice handles both null shapes. let Some(blinding_inverse) = (unsafe { input_slice(blinding_inverse, blinding_inverse_len) }) else { return usize::MAX; }; @@ -1430,22 +1967,42 @@ pub extern "C" fn ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding( }; let mut scratch = key.private_scratch(); key - .decrypt_pkcs1v15_with_blinding_factor_and_scratch(ciphertext, blinding_factor, blinding_inverse, out, &mut scratch) + .decrypt_pkcs1v15_with_blinding_factor_and_scratch( + ciphertext, + RsaBlindingPair::new(blinding_factor, blinding_inverse), + out, + &mut scratch, + ) .unwrap_or(usize::MAX) } +/// Parse and re-encode the CT fixture RSA private key through the C ABI harness. +/// +/// # Safety +/// +/// - A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer must +/// reference one allocation of `out_len` initialized writable bytes, `out_len` must not exceed +/// `isize::MAX`, and the range must be exclusively accessed for the call. +/// - A null `pkcs8_der` pointer is permitted only when `pkcs8_der_len` is zero. A non-null pointer +/// must reference one allocation of `pkcs8_der_len` initialized bytes, `pkcs8_der_len` must not +/// exceed `isize::MAX`, and the range must remain immutable for the call. +/// - A nonempty output range must be disjoint from the input range. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_rsa_private_key_pkcs8_roundtrip( +pub unsafe extern "C" fn ct_entry_rsa_private_key_pkcs8_roundtrip( out: *mut u8, out_len: usize, pkcs8_der: *const u8, pkcs8_der_len: usize, ) -> usize { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes out's allocation, length bound, initialization, + // exclusivity, lifetime, and disjointness from the live input; output_slice handles both null + // shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null pkcs8_der; input_slice + // handles both null shapes. let Some(pkcs8_der) = (unsafe { input_slice(pkcs8_der, pkcs8_der_len) }) else { return usize::MAX; }; @@ -1463,13 +2020,37 @@ pub extern "C" fn ct_entry_rsa_private_key_pkcs8_roundtrip( macro_rules! blake2_keyed_entry { ($name:ident, $ty:ty, $key_ty:ty, $out_len:literal) => { + #[doc = concat!("Compute a keyed `", stringify!($ty), "` digest through the CT harness.")] + /// + /// # Safety + /// + /// - A null `key` or `data` pointer represents an empty slice only when its corresponding + /// length is zero. Every non-null pointer must reference one allocation of initialized + /// bytes, its length must not exceed `isize::MAX`, and the range must remain immutable + /// through its last use. The two read-only ranges may overlap. + #[doc = concat!( + "- When non-null, `out` must be valid for writes of ", + stringify!($out_len), + " bytes. A null output is rejected. The output may overlap caller input because hashing and input borrows end ", + "before the digest is copied out; it must not overlap the stack-owned digest." + )] #[unsafe(no_mangle)] - pub extern "C" fn $name(key: *const u8, key_len: usize, data: *const u8, data_len: usize, out: *mut u8) -> u8 { - // SAFETY: FFI input pointers are validated by slice helpers. + pub unsafe extern "C" fn $name( + key: *const u8, + key_len: usize, + data: *const u8, + data_len: usize, + out: *mut u8, + ) -> u8 { + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null key; input_slice handles + // both null shapes. let Some(key) = (unsafe { input_slice(key, key_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null data; input_slice handles + // both null shapes. Shared key and data ranges may overlap. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; @@ -1478,7 +2059,9 @@ macro_rules! blake2_keyed_entry { return STATUS_ERR; }; let digest = <$ty>::keyed_digest(key, data); - // SAFETY: The output pointer must reference exactly `$out_len` writable bytes. + // SAFETY: The generated function contract requires non-null out to be valid for writes of + // the selected digest width. Hashing and the caller-input borrows have ended, and the + // contract prevents overlap with the stack-owned digest copied by write_array. if unsafe { write_array::<$out_len>(out, &digest) } { STATUS_OK } else { @@ -1493,19 +2076,41 @@ blake2_keyed_entry!(ct_entry_blake2b512_keyed_digest, Blake2b512, Blake2bKey, 64 blake2_keyed_entry!(ct_entry_blake2s128_keyed_digest, Blake2s128, Blake2sKey, 16); blake2_keyed_entry!(ct_entry_blake2s256_keyed_digest, Blake2s256, Blake2sKey, 32); +/// Compute a keyed BLAKE3 digest through the CT harness. +/// +/// # Safety +/// +/// - When non-null, `key` must be valid for reads of 32 initialized bytes and remain immutable +/// while copied. A null key is rejected. +/// - A null `data` pointer represents an empty slice only when `data_len` is zero. A non-null +/// pointer must reference one allocation of `data_len` initialized bytes, `data_len` must not +/// exceed `isize::MAX`, and the range must remain immutable through its last use. The key and +/// data ranges may overlap. +/// - When non-null, `out` must be valid for writes of 32 bytes. A null output is rejected. Output +/// may overlap caller input because hashing and input borrows end before the digest is copied +/// out; it must not overlap the stack-owned digest. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_blake3_keyed_digest(key: *const u8, data: *const u8, data_len: usize, out: *mut u8) -> u8 { - // SAFETY: Fixed-size FFI input is copied by value after null check. +pub unsafe extern "C" fn ct_entry_blake3_keyed_digest( + key: *const u8, + data: *const u8, + data_len: usize, + out: *mut u8, +) -> u8 { + // SAFETY: The function contract requires non-null key to expose 32 initialized, readable bytes + // that remain immutable while read_array copies them into owned storage. let Some(key) = (unsafe { read_array::<32>(key) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointer is validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null data; input_slice handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; let digest = Blake3::keyed_digest(&key, data); - // SAFETY: The output pointer must reference exactly 32 writable bytes. + // SAFETY: The function contract requires non-null out to be valid for a 32-byte write. Hashing + // and caller-input borrows have ended, and the contract prevents overlap with the stack-owned + // digest copied by write_array. if unsafe { write_array(out, digest.as_bytes()) } { STATUS_OK } else { @@ -1513,14 +2118,27 @@ pub extern "C" fn ct_entry_blake3_keyed_digest(key: *const u8, data: *const u8, } } +/// Compute a public-data SHA-256 digest through the CT harness. +/// +/// # Safety +/// +/// - A null `data` pointer represents an empty slice only when `data_len` is zero. A non-null +/// pointer must reference one allocation of `data_len` initialized bytes, `data_len` must not +/// exceed `isize::MAX`, and the range must remain immutable through its last use. +/// - When non-null, `out` must be valid for writes of 32 bytes. A null output is rejected. Output +/// may overlap the input because hashing and the input borrow end before the digest is copied +/// out; it must not overlap the stack-owned digest. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_public_sha256_digest(data: *const u8, data_len: usize, out: *mut u8) -> u8 { - // SAFETY: FFI input pointer is validated by `input_slice`. +pub unsafe extern "C" fn ct_entry_public_sha256_digest(data: *const u8, data_len: usize, out: *mut u8) -> u8 { + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null data; input_slice handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; let digest = Sha256::digest(data); - // SAFETY: The output pointer must reference exactly 32 writable bytes. + // SAFETY: The function contract requires non-null out to be valid for a 32-byte write. Hashing + // and the caller-input borrow have ended, and the contract prevents overlap with the + // stack-owned digest copied by write_array. if unsafe { write_array(out, &digest) } { STATUS_OK } else { @@ -1528,9 +2146,17 @@ pub extern "C" fn ct_entry_public_sha256_digest(data: *const u8, data_len: usize } } +/// Compute a public-data IEEE CRC-32 checksum through the CT harness. +/// +/// # Safety +/// +/// A null `data` pointer represents an empty slice only when `data_len` is zero. A non-null +/// pointer must reference one allocation of `data_len` initialized bytes, `data_len` must not +/// exceed `isize::MAX`, and the range must remain immutable for the call. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_public_crc32_checksum(data: *const u8, data_len: usize) -> u32 { - // SAFETY: FFI input pointer is validated by `input_slice`. +pub unsafe extern "C" fn ct_entry_public_crc32_checksum(data: *const u8, data_len: usize) -> u32 { + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null data; input_slice handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return 0; }; diff --git a/tools/ct-harness/src/main.rs b/tools/ct-harness/src/main.rs index b5426873..e31f4041 100644 --- a/tools/ct-harness/src/main.rs +++ b/tools/ct-harness/src/main.rs @@ -3,7 +3,7 @@ fn main() { macro_rules! retain { ($($entry:path),+ $(,)?) => { - $(let _ = std::hint::black_box($entry as *const ());)+ + $(let _ = core::hint::black_box($entry as *const ());)+ }; } diff --git a/tools/wasm-runtime-vectors/Cargo.lock b/tools/wasm-runtime-vectors/Cargo.lock index 290a790e..5ce59554 100644 --- a/tools/wasm-runtime-vectors/Cargo.lock +++ b/tools/wasm-runtime-vectors/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-wasm-runtime-vectors" diff --git a/tools/wasm-runtime-vectors/src/main.rs b/tools/wasm-runtime-vectors/src/main.rs index 1443bc25..4854c89a 100644 --- a/tools/wasm-runtime-vectors/src/main.rs +++ b/tools/wasm-runtime-vectors/src/main.rs @@ -1,25 +1,31 @@ use rscrypto::{Blake2b512, Blake3, Digest, Sha256, Sha512}; -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } fn assert_hex(actual: &[u8], expected: &str) { assert_eq!(actual.len().strict_mul(2), expected.len()); for (i, chunk) in expected.as_bytes().chunks_exact(2).enumerate() { - let byte = (hex_value(chunk[0]) << 4) | hex_value(chunk[1]); + let high = hex_value(chunk[0]).expect("known hash vector must contain hexadecimal digits"); + let low = hex_value(chunk[1]).expect("known hash vector must contain hexadecimal digits"); + let byte = high.strict_shl(4) | low; assert_eq!(actual[i], byte, "hex mismatch at byte {i}"); } } fn patterned_bytes(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(37).wrapping_add((i >> 8) as u8)) + .map(|i| { + i.to_le_bytes()[0] + .wrapping_mul(37) + .wrapping_add(i.strict_shr(8).to_le_bytes()[0]) + }) .collect() } From b18e26c0e6222d88c49a4d9e4beaa3693bb41918 Mon Sep 17 00:00:00 2001 From: LoadingALIAS Date: Mon, 17 Aug 2026 00:23:54 -0400 Subject: [PATCH 02/12] ci: make assurance failures definitive Pin Nextest and disable retries so test failures remain visible. Reject yanked crates against the locked dependency graph. --- .config/nextest.toml | 4 +++- deny.toml | 2 +- scripts/ci/dependabot-smoke.sh | 2 +- scripts/ci/release-preflight.sh | 2 +- scripts/ci/run-rust-job.sh | 4 ++-- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 91ab73c0..f139574f 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,3 +1,5 @@ +nextest-version = { required = "0.9.143", recommended = "0.9.143" } + [profile.default] status-level = "pass" success-output = "never" @@ -26,7 +28,7 @@ failure-output = "immediate-final" fail-fast = false test-threads = "num-cpus" slow-timeout = { period = "60s", terminate-after = 4 } -retries = { backoff = "exponential", count = 2, delay = "1s", jitter = true } +retries = 0 [profile.commit.junit] path = "junit.xml" diff --git a/deny.toml b/deny.toml index 3bfaef86..bfe406b2 100644 --- a/deny.toml +++ b/deny.toml @@ -21,7 +21,7 @@ targets = [ ] [advisories] -yanked = "warn" +yanked = "deny" # RustCrypto `rsa` is a dev-only differential oracle and benchmark baseline. # rscrypto never exposes it in production paths; production RSA verification # uses `src/auth/rsa.rs`. Keep this scoped to the known Marvin advisory until diff --git a/scripts/ci/dependabot-smoke.sh b/scripts/ci/dependabot-smoke.sh index 14ec2b8e..7863c393 100755 --- a/scripts/ci/dependabot-smoke.sh +++ b/scripts/ci/dependabot-smoke.sh @@ -39,7 +39,7 @@ run_root_smoke() { cargo fetch --locked cargo check --locked --workspace --all-targets --all-features cargo test --locked --workspace --all-features --no-run - cargo deny check advisories + cargo deny --locked check advisories } run_manifest_smoke() { diff --git a/scripts/ci/release-preflight.sh b/scripts/ci/release-preflight.sh index 853bf515..7bc2b3e1 100755 --- a/scripts/ci/release-preflight.sh +++ b/scripts/ci/release-preflight.sh @@ -110,7 +110,7 @@ cargo rail config validate --strict cargo rail config migrate --check # Exact-commit Weekly release mode owns exhaustive compiler-backed Cargo graph # assurance. The release evidence gate verifies that named job before publication. -cargo deny check all +cargo deny --locked check all # RustCrypto `rsa` is used only as a dev/test/bench oracle. Production RSA # verification is implemented in `src/auth/rsa.rs`; keep this scoped to the # known Marvin advisory until the oracle dependency is removed or fixed. diff --git a/scripts/ci/run-rust-job.sh b/scripts/ci/run-rust-job.sh index a5a77cb4..f0a25a96 100755 --- a/scripts/ci/run-rust-job.sh +++ b/scripts/ci/run-rust-job.sh @@ -186,13 +186,13 @@ run_supply_chain() { require_one_of supply_chain_mode "$mode" light full if [[ "$mode" == "full" ]]; then - cargo deny check all + cargo deny --locked check all # RustCrypto `rsa` is used only as a dev/test/bench oracle. Production RSA # verification is implemented in `src/auth/rsa.rs`; keep this scoped to the # known Marvin advisory until the oracle dependency is removed or fixed. cargo audit --ignore RUSTSEC-2023-0071 else - cargo deny check advisories + cargo deny --locked check advisories fi } From e5cef85ff908b419bf15a15bed0287c3e3da2e1d Mon Sep 17 00:00:00 2001 From: LoadingALIAS Date: Mon, 17 Aug 2026 00:24:04 -0400 Subject: [PATCH 03/12] ci: restore strict cross-target validation Activate each resolved toolchain through RUSTUP_TOOLCHAIN so repository overrides cannot substitute development Rust. Keep platform and feature-specific checks warning-free under their actual compiler contracts. --- .github/actions/setup-toolchain/action.yaml | 5 +- scripts/ci/check-ci-ownership-test.sh | 7 +++ scripts/ci/check-ci-ownership.sh | 7 +++ scripts/ci/setup-toolchain.sh | 1 - scripts/ci/tool-integrity-test.sh | 8 +++- src/aead/aes.rs | 51 +++++++++++++++------ src/aead/chacha20poly1305.rs | 13 ++++-- src/aead/polyval.rs | 2 +- src/checksum/kernel_table.rs | 2 +- src/hashes/crypto/blake3/dispatch_tables.rs | 8 ++-- src/hashes/crypto/keccak/kernel_test.rs | 8 ++-- src/lib.rs | 21 ++++++++- src/platform/detect/tests.rs | 34 ++++++++------ tests/platform_amx_permission.rs | 8 +--- 14 files changed, 120 insertions(+), 55 deletions(-) diff --git a/.github/actions/setup-toolchain/action.yaml b/.github/actions/setup-toolchain/action.yaml index caf2ee10..f3840606 100644 --- a/.github/actions/setup-toolchain/action.yaml +++ b/.github/actions/setup-toolchain/action.yaml @@ -40,4 +40,7 @@ runs: env: TOOLCHAIN: ${{ steps.read-toolchain.outputs.toolchain }} TOOLCHAIN_COMPONENTS: ${{ inputs.components }} - run: scripts/ci/setup-toolchain.sh "$TOOLCHAIN" "$TOOLCHAIN_COMPONENTS" + run: | + scripts/ci/setup-toolchain.sh "$TOOLCHAIN" "$TOOLCHAIN_COMPONENTS" + echo "RUSTUP_TOOLCHAIN=$TOOLCHAIN" >> "$GITHUB_ENV" + RUSTUP_TOOLCHAIN="$TOOLCHAIN" rustc --version --verbose diff --git a/scripts/ci/check-ci-ownership-test.sh b/scripts/ci/check-ci-ownership-test.sh index c0298ad2..8e900111 100755 --- a/scripts/ci/check-ci-ownership-test.sh +++ b/scripts/ci/check-ci-ownership-test.sh @@ -166,6 +166,13 @@ printf '\n - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca002889 >>"$unauthenticated_rustup/.github/actions/setup-toolchain/action.yaml" expect_failure "$unauthenticated_rustup" "toolchain setup can run a network bootstrap installer" +inactive_toolchain_contract="$TMP_ROOT/inactive-toolchain-contract" +make_fixture "$inactive_toolchain_contract" +sed -i.bak '/echo "RUSTUP_TOOLCHAIN=.*GITHUB_ENV"/d' \ + "$inactive_toolchain_contract/.github/actions/setup-toolchain/action.yaml" +rm -f "$inactive_toolchain_contract/.github/actions/setup-toolchain/action.yaml.bak" +expect_failure "$inactive_toolchain_contract" "toolchain contract is installed but not activated" + floating_rail_action="$TMP_ROOT/floating-rail-action" make_fixture "$floating_rail_action" yq eval '(.jobs."rail-plan".steps[] | select(.id == "rail") | .uses) = "loadingalias/cargo-rail-action@v6"' -i \ diff --git a/scripts/ci/check-ci-ownership.sh b/scripts/ci/check-ci-ownership.sh index e9aa93a2..af02963e 100755 --- a/scripts/ci/check-ci-ownership.sh +++ b/scripts/ci/check-ci-ownership.sh @@ -346,6 +346,13 @@ release_intent_condition=$(yq eval '.jobs."rail-plan".steps[] | select(.name == || fail "only repository-owned Cargo Rail release PRs may consume change intent" grep -Fq 'scripts/ci/setup-toolchain.sh "$TOOLCHAIN" "$TOOLCHAIN_COMPONENTS"' "$TOOLCHAIN_ACTION" \ || fail "toolchain setup must use the repository-owned rustup policy" +grep -Fq 'echo "RUSTUP_TOOLCHAIN=$TOOLCHAIN" >> "$GITHUB_ENV"' "$TOOLCHAIN_ACTION" \ + || fail "toolchain setup must activate the resolved contract for later steps" +grep -Fq 'RUSTUP_TOOLCHAIN="$TOOLCHAIN" rustc --version --verbose' "$TOOLCHAIN_ACTION" \ + || fail "toolchain setup must verify the activated contract without a rust-toolchain override" +if grep -Fq 'rustup default ' "$SETUP_TOOLCHAIN"; then + fail "toolchain setup must not mutate a runner-global default" +fi if grep -Eq '[.]cargo/(bin|[.]crates)|[.]opam' "$SETUP_ACTION"; then fail "CI tool executables and OPAM switches must not be restored from caches" fi diff --git a/scripts/ci/setup-toolchain.sh b/scripts/ci/setup-toolchain.sh index beba45ec..ec00eacc 100755 --- a/scripts/ci/setup-toolchain.sh +++ b/scripts/ci/setup-toolchain.sh @@ -34,5 +34,4 @@ if [[ -n "$components" ]]; then fi rustup "${install_args[@]}" -rustup default "$toolchain" rustc "+$toolchain" --version --verbose diff --git a/scripts/ci/tool-integrity-test.sh b/scripts/ci/tool-integrity-test.sh index 126c0dac..6d204c88 100755 --- a/scripts/ci/tool-integrity-test.sh +++ b/scripts/ci/tool-integrity-test.sh @@ -333,6 +333,7 @@ SH cat >"$package_bin/rustc" <<'SH' #!/usr/bin/env bash +printf 'rustc %s\n' "$*" >>"$MOCK_PACKAGE_LOG" printf 'rustc 1.99.0-nightly\ncommit-date: 2026-07-16\n' SH @@ -605,8 +606,11 @@ MOCK_PACKAGE_LOG="$package_log" PATH="$package_bin:$PATH" \ grep -Fq \ 'rustup toolchain install nightly-2026-08-12 --profile minimal --no-self-update --component clippy --component rustfmt' \ "$package_log" || fail "rustup toolchain command was not exact" -grep -Fq 'rustup default nightly-2026-08-12' "$package_log" \ - || fail "rustup did not select the exact toolchain" +grep -Fq 'rustc +nightly-2026-08-12 --version --verbose' "$package_log" \ + || fail "installed toolchain was not verified explicitly" +if grep -Fq 'rustup default ' "$package_log"; then + fail "toolchain setup mutated the runner default" +fi if MOCK_PACKAGE_LOG="$package_log" PATH="$package_bin:$PATH" \ "$REPO_ROOT/scripts/ci/setup-toolchain.sh" nightly clippy >/dev/null 2>&1; then fail "mutable rustup channel was accepted" diff --git a/src/aead/aes.rs b/src/aead/aes.rs index 2b5d572d..173b1713 100644 --- a/src/aead/aes.rs +++ b/src/aead/aes.rs @@ -2700,6 +2700,8 @@ unsafe fn x86_gcm_ctr_blocks_be_16( #[cfg(all(target_arch = "x86_64", feature = "aes-gcm", test))] #[target_feature(enable = "avx2")] #[inline] +/// # Safety +/// Caller must ensure AVX2 is available. unsafe fn x86_gcm_ctr_blocks_be_2(iv_words: [u32; 3], ctr: u32) -> core::arch::x86_64::__m256i { use core::arch::x86_64::*; @@ -5314,14 +5316,11 @@ mod tests { #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] fn fill_expected_gcm_counter_blocks(iv_prefix: &[u8; 12], ctr: u32, expected: &mut [u8; N]) { debug_assert_eq!(N.strict_rem(BLOCK_SIZE), 0); - let blocks = N.strict_div(BLOCK_SIZE); - let mut block_idx = 0usize; - while block_idx < blocks { - let start = block_idx.strict_mul(BLOCK_SIZE); - expected[start..start.strict_add(12)].copy_from_slice(iv_prefix); - expected[start.strict_add(12)..start.strict_add(16)] - .copy_from_slice(&ctr.wrapping_add(block_idx as u32).to_be_bytes()); - block_idx = block_idx.strict_add(1); + let (blocks, tail) = expected.as_chunks_mut::(); + debug_assert!(tail.is_empty()); + for (block_idx, block) in (0u32..).zip(blocks) { + block[..12].copy_from_slice(iv_prefix); + block[12..].copy_from_slice(&ctr.wrapping_add(block_idx).to_be_bytes()); } } @@ -5494,7 +5493,11 @@ mod tests { } } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] + #[cfg(all( + target_arch = "x86_64", + feature = "aes-gcm", + any(target_os = "linux", target_os = "macos") + ))] fn x86_z512_gcm_caps_available() -> bool { let required = crate::platform::caps::x86::VAES_READY | crate::platform::caps::x86::VPCLMUL_READY @@ -5502,7 +5505,11 @@ mod tests { crate::platform::caps().has(required) } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] + #[cfg(all( + target_arch = "x86_64", + feature = "aes-gcm", + any(target_os = "linux", target_os = "macos") + ))] struct X86GcmTestPowers16 { h_polyval: u128, h_powers_rev: [u128; 4], @@ -5515,7 +5522,11 @@ mod tests { h_powers_rev_128: [u128; 128], } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] + #[cfg(all( + target_arch = "x86_64", + feature = "aes-gcm", + any(target_os = "linux", target_os = "macos") + ))] impl X86GcmTestPowers16 { fn tables<'a>(&'a self) -> X86GcmTables<'a> { X86GcmTables { @@ -5532,7 +5543,11 @@ mod tests { } } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] + #[cfg(all( + target_arch = "x86_64", + feature = "aes-gcm", + any(target_os = "linux", target_os = "macos") + ))] fn x86_gcm_test_powers_16() -> X86GcmTestPowers16 { let h_polyval = 0x1287_3d5b_fedc_ba09_7654_3210_f0e1_d2c3u128; let powers = crate::aead::polyval::precompute_powers_128(h_polyval); @@ -5557,7 +5572,11 @@ mod tests { } } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] + #[cfg(all( + target_arch = "x86_64", + feature = "aes-gcm", + any(target_os = "linux", target_os = "macos") + ))] fn x86_gcm_wrap_counter_block() -> [u8; 16] { let mut counter = [0u8; 16]; counter[..12].copy_from_slice(b"ctr wrap iv!"); @@ -5565,7 +5584,11 @@ mod tests { counter } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] + #[cfg(all( + target_arch = "x86_64", + feature = "aes-gcm", + any(target_os = "linux", target_os = "macos") + ))] fn fill_x86_gcm_test_plaintext(out: &mut [u8]) { let mut i = 0usize; while i < out.len() { diff --git a/src/aead/chacha20poly1305.rs b/src/aead/chacha20poly1305.rs index 5c627afb..f7cc052c 100644 --- a/src/aead/chacha20poly1305.rs +++ b/src/aead/chacha20poly1305.rs @@ -992,19 +992,22 @@ mod tests { for plaintext_len in [ 1usize, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, 256, 257, 1024, ] { - let plaintext = (0..plaintext_len) - .map(|index| 0x51u8.wrapping_add((index as u8).wrapping_mul(13))) + let plaintext = (0u8..=u8::MAX) + .cycle() + .take(plaintext_len) + .map(|index| 0x51u8.wrapping_add(index.wrapping_mul(13))) .collect::>(); for aad_len in [0usize, 1, 13, 14, 15, 16, 17, 31, 32, 63, 64] { - let aad = (0..aad_len) - .map(|index| 0xa7u8.wrapping_add((index as u8).wrapping_mul(7))) + let aad = (0u8..) + .take(aad_len) + .map(|index| 0xa7u8.wrapping_add(index.wrapping_mul(7))) .collect::>(); let mut ciphertext = plaintext.clone(); let tag = cipher .encrypt_in_place_owned_unchecked(&nonce, &aad, &mut ciphertext) - .unwrap(); + .expect("bounded test input must encrypt through the owned path"); let mut actual = ciphertext.clone(); // SAFETY: the test returned unless AVX2+BMI2 are available; every selected ciphertext is nonempty and well // below ChaCha20's 2^32-block limit. diff --git a/src/aead/polyval.rs b/src/aead/polyval.rs index c5c1db61..cefe894b 100644 --- a/src/aead/polyval.rs +++ b/src/aead/polyval.rs @@ -2596,7 +2596,7 @@ mod tests { let mut data = [0u8; 321]; let mut i = 0usize; while i < len { - data[i] = i.wrapping_mul(37).wrapping_add(19) as u8; + data[i] = i.to_le_bytes()[0].wrapping_mul(37).wrapping_add(19); i = i.strict_add(1); } diff --git a/src/checksum/kernel_table.rs b/src/checksum/kernel_table.rs index f77317ff..c8dce155 100644 --- a/src/checksum/kernel_table.rs +++ b/src/checksum/kernel_table.rs @@ -3064,7 +3064,7 @@ mod tests { assert!(!VPCLMUL_READY.has(CRC32C_READY)); let table = select_crc64_table(VPCLMUL_READY); - assert!(core::ptr::eq(table, &GENERIC_X86_VPCLMUL_NO_CRC32C_TABLE)); + assert!(core::ptr::eq(table, &raw const GENERIC_X86_VPCLMUL_NO_CRC32C_TABLE)); assert_eq!(table.select_names(4096).crc64_xz_name, "x86_64/vpclmul-2way"); assert_eq!(table.select_names(4097).crc64_xz_name, "x86_64/vpclmul-4x512"); assert_eq!(table.select_names(4097).crc64_nvme_name, "x86_64/vpclmul-4way"); diff --git a/src/hashes/crypto/blake3/dispatch_tables.rs b/src/hashes/crypto/blake3/dispatch_tables.rs index 1fe518c6..55e61722 100644 --- a/src/hashes/crypto/blake3/dispatch_tables.rs +++ b/src/hashes/crypto/blake3/dispatch_tables.rs @@ -908,19 +908,19 @@ mod tests { let sapphire_rapids = x86::AVX512_READY | x86::INTEL_SAPPHIRE_RAPIDS; assert!(core::ptr::eq( select_profile_for_caps(sapphire_rapids), - &PROFILE_INTEL_SAPPHIRE_RAPIDS + &raw const PROFILE_INTEL_SAPPHIRE_RAPIDS )); assert!(core::ptr::eq( select_profile_for_caps(sapphire_rapids | ALL_AMX), - &PROFILE_INTEL_SAPPHIRE_RAPIDS + &raw const PROFILE_INTEL_SAPPHIRE_RAPIDS )); assert!(core::ptr::eq( select_profile_for_caps(x86::AVX512_READY), - &PROFILE_X86_AVX512 + &raw const PROFILE_X86_AVX512 )); assert!(core::ptr::eq( select_profile_for_caps(x86::AVX512_READY | ALL_AMX), - &PROFILE_X86_AVX512 + &raw const PROFILE_X86_AVX512 )); } } diff --git a/src/hashes/crypto/keccak/kernel_test.rs b/src/hashes/crypto/keccak/kernel_test.rs index 19d2bb58..1d0f4c5c 100644 --- a/src/hashes/crypto/keccak/kernel_test.rs +++ b/src/hashes/crypto/keccak/kernel_test.rs @@ -167,11 +167,11 @@ mod tests { for seed in 0u8..16 { let mut input_a = [0u8; 200]; let mut input_b = [0u8; 200]; - for (i, byte) in input_a.iter_mut().enumerate() { - *byte = seed.wrapping_add((i as u8).wrapping_mul(17)); + for (i, byte) in (0u8..).zip(&mut input_a) { + *byte = seed.wrapping_add(i.wrapping_mul(17)); } - for (i, byte) in input_b.iter_mut().enumerate() { - *byte = seed.wrapping_mul(3).wrapping_add((i as u8).wrapping_mul(29)); + for (i, byte) in (0u8..).zip(&mut input_b) { + *byte = seed.wrapping_mul(3).wrapping_add(i.wrapping_mul(29)); } let mut state_a = state_from_bytes(&input_a); diff --git a/src/lib.rs b/src/lib.rs index d72bb357..6200286c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -152,7 +152,26 @@ assert!( )] // s390x VGFM/hash backends use vector asm; selected checksum/hash/AEAD/password // kernels also use portable SIMD. -#![cfg_attr(target_arch = "s390x", feature(asm_experimental_reg))] +#![cfg_attr( + all( + target_arch = "s390x", + any( + feature = "crc16", + feature = "crc24", + feature = "crc32", + feature = "crc64", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "aegis256", + feature = "xxh3", + feature = "chacha20poly1305", + feature = "xchacha20poly1305", + feature = "ml-kem", + feature = "argon2" + ) + ), + feature(asm_experimental_reg) +)] #![cfg_attr( all( target_arch = "s390x", diff --git a/src/platform/detect/tests.rs b/src/platform/detect/tests.rs index 954c4cab..948bfb8b 100644 --- a/src/platform/detect/tests.rs +++ b/src/platform/detect/tests.rs @@ -427,27 +427,31 @@ mod tests { snapshot.leaf7_0.ebx |= 1 << 5; }); + enum Leaf7Register { + Ebx, + Ecx, + Edx, + } let avx512_cases = [ - (x86::AVX512DQ, "ebx", 17), - (x86::AVX512IFMA, "ebx", 21), - (x86::AVX512CD, "ebx", 28), - (x86::AVX512BW, "ebx", 30), - (x86::AVX512VL, "ebx", 31), - (x86::AVX512VBMI, "ecx", 1), - (x86::AVX512VBMI2, "ecx", 6), - (x86::AVX512VNNI, "ecx", 11), - (x86::AVX512BITALG, "ecx", 12), - (x86::AVX512VPOPCNTDQ, "ecx", 14), - (x86::AVX512VP2INTERSECT, "edx", 8), + (x86::AVX512DQ, Leaf7Register::Ebx, 17), + (x86::AVX512IFMA, Leaf7Register::Ebx, 21), + (x86::AVX512CD, Leaf7Register::Ebx, 28), + (x86::AVX512BW, Leaf7Register::Ebx, 30), + (x86::AVX512VL, Leaf7Register::Ebx, 31), + (x86::AVX512VBMI, Leaf7Register::Ecx, 1), + (x86::AVX512VBMI2, Leaf7Register::Ecx, 6), + (x86::AVX512VNNI, Leaf7Register::Ecx, 11), + (x86::AVX512BITALG, Leaf7Register::Ecx, 12), + (x86::AVX512VPOPCNTDQ, Leaf7Register::Ecx, 14), + (x86::AVX512VP2INTERSECT, Leaf7Register::Edx, 8), ]; for (feature, register, bit) in avx512_cases { assert_feature(avx512_caps() | feature, |snapshot| { enable_avx512(snapshot); match register { - "ebx" => snapshot.leaf7_0.ebx |= 1 << bit, - "ecx" => snapshot.leaf7_0.ecx |= 1 << bit, - "edx" => snapshot.leaf7_0.edx |= 1 << bit, - _ => unreachable!(), + Leaf7Register::Ebx => snapshot.leaf7_0.ebx |= 1 << bit, + Leaf7Register::Ecx => snapshot.leaf7_0.ecx |= 1 << bit, + Leaf7Register::Edx => snapshot.leaf7_0.edx |= 1 << bit, } }); } diff --git a/tests/platform_amx_permission.rs b/tests/platform_amx_permission.rs index d72d4e92..f28d08c1 100644 --- a/tests/platform_amx_permission.rs +++ b/tests/platform_amx_permission.rs @@ -18,15 +18,11 @@ const XFEATURE_XTILEDATA: usize = 18; const XCOMP_TILE_MASK: u64 = (1 << 17) | (1 << 18); fn cpu_supports_amx_tile() -> bool { - // MSRV: CPUID is unsafe on Rust 1.91 but safe on the pinned nightly. - // SAFETY: CPUID is a non-privileged x86-64 identification instruction. - let leaf0 = unsafe { core::arch::x86_64::__cpuid(0) }; + let leaf0 = core::arch::x86_64::__cpuid(0); if leaf0.eax < 7 { return false; } - // SAFETY: CPUID leaf 7, subleaf 0 is valid because leaf 0 reports support - // for leaf 7; the intrinsic only returns register values. - let leaf7 = unsafe { core::arch::x86_64::__cpuid_count(7, 0) }; + let leaf7 = core::arch::x86_64::__cpuid_count(7, 0); leaf7.edx & (1 << 24) != 0 } From 9d06ea29d2ddab4c080cbc3bbdebff7eeaf1256d Mon Sep 17 00:00:00 2001 From: LoadingALIAS Date: Mon, 17 Aug 2026 02:25:25 -0400 Subject: [PATCH 04/12] ci: isolate mocked shell fixtures Clear caller startup hooks before launching fixture subprocesses so mocked PATH entries cannot be replaced by host shell initialization. --- scripts/ci/check-action-pins-test.sh | 1 + scripts/ci/publish-immutable-release-test.sh | 1 + scripts/ci/release-evidence-check-test.sh | 1 + scripts/ci/repository-controls-evidence-test.sh | 1 + scripts/ci/run-rust-job-test.sh | 1 + 5 files changed, 5 insertions(+) diff --git a/scripts/ci/check-action-pins-test.sh b/scripts/ci/check-action-pins-test.sh index 75ddc32c..6518b288 100755 --- a/scripts/ci/check-action-pins-test.sh +++ b/scripts/ci/check-action-pins-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CHECKER="$SCRIPT_DIR/check-action-pins.sh" diff --git a/scripts/ci/publish-immutable-release-test.sh b/scripts/ci/publish-immutable-release-test.sh index 57b3828a..94d7c19d 100755 --- a/scripts/ci/publish-immutable-release-test.sh +++ b/scripts/ci/publish-immutable-release-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PUBLISHER="$SCRIPT_DIR/publish-immutable-release.sh" diff --git a/scripts/ci/release-evidence-check-test.sh b/scripts/ci/release-evidence-check-test.sh index 46bfd2f5..afc3a19a 100755 --- a/scripts/ci/release-evidence-check-test.sh +++ b/scripts/ci/release-evidence-check-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" diff --git a/scripts/ci/repository-controls-evidence-test.sh b/scripts/ci/repository-controls-evidence-test.sh index fbb91a30..c2d44c7a 100755 --- a/scripts/ci/repository-controls-evidence-test.sh +++ b/scripts/ci/repository-controls-evidence-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" diff --git a/scripts/ci/run-rust-job-test.sh b/scripts/ci/run-rust-job-test.sh index 4839918b..d5c59373 100755 --- a/scripts/ci/run-rust-job-test.sh +++ b/scripts/ci/run-rust-job-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DISPATCHER="$SCRIPT_DIR/run-rust-job.sh" From d9b23690c575fab0f9d4a571422c90c7bebd393a Mon Sep 17 00:00:00 2001 From: LoadingALIAS Date: Mon, 17 Aug 2026 02:25:32 -0400 Subject: [PATCH 05/12] ci: harden toolchain environment activation Write the validated exact toolchain through the repository-owned setup script so action inputs never reach GitHub environment-file syntax. Extend ownership and integrity checks for that boundary. --- .github/actions/setup-toolchain/action.yaml | 3 +-- scripts/ci/check-ci-ownership-test.sh | 2 +- scripts/ci/check-ci-ownership.sh | 4 ++-- scripts/ci/setup-toolchain.sh | 8 ++++++-- scripts/ci/tool-integrity-test.sh | 6 +++++- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/actions/setup-toolchain/action.yaml b/.github/actions/setup-toolchain/action.yaml index f3840606..67e7ae40 100644 --- a/.github/actions/setup-toolchain/action.yaml +++ b/.github/actions/setup-toolchain/action.yaml @@ -41,6 +41,5 @@ runs: TOOLCHAIN: ${{ steps.read-toolchain.outputs.toolchain }} TOOLCHAIN_COMPONENTS: ${{ inputs.components }} run: | - scripts/ci/setup-toolchain.sh "$TOOLCHAIN" "$TOOLCHAIN_COMPONENTS" - echo "RUSTUP_TOOLCHAIN=$TOOLCHAIN" >> "$GITHUB_ENV" + scripts/ci/setup-toolchain.sh "$TOOLCHAIN" "$TOOLCHAIN_COMPONENTS" "$GITHUB_ENV" RUSTUP_TOOLCHAIN="$TOOLCHAIN" rustc --version --verbose diff --git a/scripts/ci/check-ci-ownership-test.sh b/scripts/ci/check-ci-ownership-test.sh index 8e900111..5838cf1d 100755 --- a/scripts/ci/check-ci-ownership-test.sh +++ b/scripts/ci/check-ci-ownership-test.sh @@ -168,7 +168,7 @@ expect_failure "$unauthenticated_rustup" "toolchain setup can run a network boot inactive_toolchain_contract="$TMP_ROOT/inactive-toolchain-contract" make_fixture "$inactive_toolchain_contract" -sed -i.bak '/echo "RUSTUP_TOOLCHAIN=.*GITHUB_ENV"/d' \ +sed -i.bak 's/ "\$GITHUB_ENV"$//' \ "$inactive_toolchain_contract/.github/actions/setup-toolchain/action.yaml" rm -f "$inactive_toolchain_contract/.github/actions/setup-toolchain/action.yaml.bak" expect_failure "$inactive_toolchain_contract" "toolchain contract is installed but not activated" diff --git a/scripts/ci/check-ci-ownership.sh b/scripts/ci/check-ci-ownership.sh index af02963e..5c102499 100755 --- a/scripts/ci/check-ci-ownership.sh +++ b/scripts/ci/check-ci-ownership.sh @@ -344,9 +344,9 @@ release_intent_condition=$(yq eval '.jobs."rail-plan".steps[] | select(.name == [[ "$release_intent_condition" == *"startsWith(github.head_ref, 'rail/release-')"* \ && "$release_intent_condition" == *"github.event.pull_request.head.repo.full_name == github.repository"* ]] \ || fail "only repository-owned Cargo Rail release PRs may consume change intent" -grep -Fq 'scripts/ci/setup-toolchain.sh "$TOOLCHAIN" "$TOOLCHAIN_COMPONENTS"' "$TOOLCHAIN_ACTION" \ +grep -Fq 'scripts/ci/setup-toolchain.sh "$TOOLCHAIN" "$TOOLCHAIN_COMPONENTS" "$GITHUB_ENV"' "$TOOLCHAIN_ACTION" \ || fail "toolchain setup must use the repository-owned rustup policy" -grep -Fq 'echo "RUSTUP_TOOLCHAIN=$TOOLCHAIN" >> "$GITHUB_ENV"' "$TOOLCHAIN_ACTION" \ +grep -Fq "printf 'RUSTUP_TOOLCHAIN=%s\\n' \"\$toolchain\" >>\"\$github_env\"" "$SETUP_TOOLCHAIN" \ || fail "toolchain setup must activate the resolved contract for later steps" grep -Fq 'RUSTUP_TOOLCHAIN="$TOOLCHAIN" rustc --version --verbose' "$TOOLCHAIN_ACTION" \ || fail "toolchain setup must verify the activated contract without a rust-toolchain override" diff --git a/scripts/ci/setup-toolchain.sh b/scripts/ci/setup-toolchain.sh index ec00eacc..688675cf 100755 --- a/scripts/ci/setup-toolchain.sh +++ b/scripts/ci/setup-toolchain.sh @@ -1,13 +1,14 @@ #!/usr/bin/env bash set -euo pipefail -if [[ $# -lt 1 || $# -gt 2 ]]; then - echo "usage: setup-toolchain.sh [comma-separated-components]" >&2 +if [[ $# -lt 1 || $# -gt 3 ]]; then + echo "usage: setup-toolchain.sh [comma-separated-components] [github-env]" >&2 exit 2 fi toolchain=$1 components=${2:-} +github_env=${3:-} if [[ ! "$toolchain" =~ ^(nightly|beta)-[0-9]{4}-[0-9]{2}-[0-9]{2}$ \ && ! "$toolchain" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then @@ -34,4 +35,7 @@ if [[ -n "$components" ]]; then fi rustup "${install_args[@]}" +if [[ -n "$github_env" ]]; then + printf 'RUSTUP_TOOLCHAIN=%s\n' "$toolchain" >>"$github_env" +fi rustc "+$toolchain" --version --verbose diff --git a/scripts/ci/tool-integrity-test.sh b/scripts/ci/tool-integrity-test.sh index 6d204c88..ef0c6418 100755 --- a/scripts/ci/tool-integrity-test.sh +++ b/scripts/ci/tool-integrity-test.sh @@ -600,14 +600,18 @@ case "$trusted_bin" in esac : >"$package_log" +toolchain_env="$TMP_ROOT/toolchain.env" +: >"$toolchain_env" MOCK_PACKAGE_LOG="$package_log" PATH="$package_bin:$PATH" \ "$REPO_ROOT/scripts/ci/setup-toolchain.sh" \ - nightly-2026-08-12 'clippy, rustfmt' >/dev/null + nightly-2026-08-12 'clippy, rustfmt' "$toolchain_env" >/dev/null grep -Fq \ 'rustup toolchain install nightly-2026-08-12 --profile minimal --no-self-update --component clippy --component rustfmt' \ "$package_log" || fail "rustup toolchain command was not exact" grep -Fq 'rustc +nightly-2026-08-12 --version --verbose' "$package_log" \ || fail "installed toolchain was not verified explicitly" +grep -Fxq 'RUSTUP_TOOLCHAIN=nightly-2026-08-12' "$toolchain_env" \ + || fail "installed toolchain was not activated for later CI steps" if grep -Fq 'rustup default ' "$package_log"; then fail "toolchain setup mutated the runner default" fi From 15a076192a3dae081c600535c19ccc931832edb9 Mon Sep 17 00:00:00 2001 From: LoadingALIAS Date: Mon, 17 Aug 2026 12:56:24 -0400 Subject: [PATCH 06/12] crypto: keep target-specific builds lint-clean --- src/aead/aes.rs | 4 ++-- src/aead/chacha20/aarch64_neon.rs | 15 ++++++--------- src/aead/chacha20poly1305.rs | 18 ++++++++---------- src/aead/poly1305/aarch64_neon.rs | 4 +++- src/auth/curve25519_edwards.rs | 11 +++++++++++ src/auth/ed25519/point_avx2.rs | 14 ++++++++++++++ src/hashes/crypto/blake3/x86_64.rs | 10 ++++++++-- src/hashes/fast/xxh3/dispatch.rs | 2 +- src/platform/detect/arch/aarch64.rs | 3 ++- 9 files changed, 55 insertions(+), 26 deletions(-) diff --git a/src/aead/aes.rs b/src/aead/aes.rs index 173b1713..02f22da8 100644 --- a/src/aead/aes.rs +++ b/src/aead/aes.rs @@ -49,7 +49,7 @@ mod ppc; #[cfg(target_arch = "riscv64")] #[path = "aes/riscv64_aes.rs"] mod rv_aes; -#[cfg(any(target_arch = "riscv64", test))] +#[cfg(any(target_arch = "riscv64", all(test, not(target_arch = "s390x"))))] #[path = "aes/riscv64_fixslice_aes.rs"] mod rv_fixslice_aes; #[cfg(target_arch = "riscv64")] @@ -1706,7 +1706,7 @@ pub(crate) fn aes256_encrypt_block(ek: &Aes256EncKey, block: &mut [u8; BLOCK_SIZ } } -#[cfg(any(target_arch = "riscv64", test))] +#[cfg(any(target_arch = "riscv64", all(test, not(target_arch = "s390x"))))] #[inline] pub(super) fn aes_enc_round_4_fixslice(blocks: &mut [[u8; BLOCK_SIZE]; 4], round_keys: &[[u8; BLOCK_SIZE]; 4]) { rv_fixslice_aes::cipher_round_4(blocks, round_keys); diff --git a/src/aead/chacha20/aarch64_neon.rs b/src/aead/chacha20/aarch64_neon.rs index 4b105f72..1c23e37e 100644 --- a/src/aead/chacha20/aarch64_neon.rs +++ b/src/aead/chacha20/aarch64_neon.rs @@ -51,8 +51,8 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: let n15 = vdupq_n_u32(load_u32_le(&nonce[8..12])); let mut counter = initial_counter; - let mut double_batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH * 2); - for chunk in &mut double_batches { + let (double_batches, double_remainder) = buffer.as_chunks_mut::<{ BLOCK_SIZE * BLOCKS_PER_BATCH * 2 }>(); + for chunk in double_batches { debug_assert!(counter.checked_add(COUNTERS_PER_DOUBLE_BATCH.strict_sub(1)).is_some()); let mut x0 = c0; @@ -175,7 +175,7 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: let ptr = chunk.as_mut_ptr(); // SAFETY: vector transpose and XOR stores because: - // 1. `chunk` is exactly eight ChaCha20 blocks from `chunks_exact_mut`. + // 1. `chunk` is an array of exactly eight ChaCha20 blocks. // 2. The first four-block group starts at `ptr`; the second starts at `ptr + 256`. // 3. NEON is guaranteed by the enclosing `#[target_feature(enable = "neon")]`. unsafe { @@ -194,10 +194,8 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: counter = counter.wrapping_add(COUNTERS_PER_DOUBLE_BATCH); } - let mut batches = double_batches - .into_remainder() - .chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); - for chunk in &mut batches { + let (batches, remainder) = double_remainder.as_chunks_mut::<{ BLOCK_SIZE * BLOCKS_PER_BATCH }>(); + for chunk in batches { debug_assert!(counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); let mut x0 = c0; @@ -285,7 +283,7 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: let ptr = chunk.as_mut_ptr(); // SAFETY: vector transpose and XOR stores because: - // 1. `chunk` is exactly `BLOCKS_PER_BATCH * BLOCK_SIZE` bytes from `chunks_exact_mut`. + // 1. `chunk` is an array of exactly `BLOCKS_PER_BATCH * BLOCK_SIZE` bytes. // 2. Each call stores four 16-byte word groups at offsets inside the 256-byte chunk. // 3. NEON is guaranteed by the enclosing `#[target_feature(enable = "neon")]`. unsafe { @@ -298,7 +296,6 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: counter = counter.wrapping_add(COUNTERS_PER_BATCH); } - let remainder = batches.into_remainder(); if !remainder.is_empty() { xor_keystream_portable(key, counter, nonce, remainder); } diff --git a/src/aead/chacha20poly1305.rs b/src/aead/chacha20poly1305.rs index f7cc052c..7faecdd4 100644 --- a/src/aead/chacha20poly1305.rs +++ b/src/aead/chacha20poly1305.rs @@ -496,17 +496,16 @@ impl ChaCha20Poly1305 { authenticator.update_padded_segment(aad); let mut counter = 1u32; - let mut chunks = buffer.chunks_exact_mut(AARCH64_INTERLEAVED_CHUNK); - for chunk in &mut chunks { + let (chunks, remainder) = buffer.as_chunks_mut::(); + for chunk in chunks { // SAFETY: the capability gate proves NEON. Every caller reaches this private helper after the public or - // diagnostic length bound, or from a same-module test with a bounded buffer; the iterator and exact counter - // advance preserve that whole-buffer bound for this segment. + // diagnostic length bound, or from a same-module test with a bounded buffer; the fixed-size chunks and exact + // counter advance preserve that whole-buffer bound for this segment. unsafe { chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), chunk) }; authenticator.update_padded_segment(chunk); counter = counter.wrapping_add(AARCH64_INTERLEAVED_BLOCKS); } - let remainder = chunks.into_remainder(); if !remainder.is_empty() { // SAFETY: the capability gate proves NEON. Every caller reaches this private helper after the public or // diagnostic length bound, or from a same-module test with a bounded buffer; `counter` tracks the preceding @@ -553,17 +552,16 @@ impl ChaCha20Poly1305 { authenticator.update_padded_segment(aad); let mut counter = 1u32; - let mut chunks = buffer.chunks_exact_mut(AARCH64_INTERLEAVED_CHUNK); - for chunk in &mut chunks { + let (chunks, remainder) = buffer.as_chunks_mut::(); + for chunk in chunks { authenticator.update_padded_segment(chunk); // SAFETY: the capability gate proves NEON. Every caller reaches this private helper after the public or - // diagnostic length bound, or from a same-module test with a bounded buffer; the iterator and exact counter - // advance preserve that whole-buffer bound for this segment. + // diagnostic length bound, or from a same-module test with a bounded buffer; the fixed-size chunks and exact + // counter advance preserve that whole-buffer bound for this segment. unsafe { chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), chunk) }; counter = counter.wrapping_add(AARCH64_INTERLEAVED_BLOCKS); } - let remainder = chunks.into_remainder(); if !remainder.is_empty() { authenticator.update_padded_segment(remainder); // SAFETY: the capability gate proves NEON. Every caller reaches this private helper after the public or diff --git a/src/aead/poly1305/aarch64_neon.rs b/src/aead/poly1305/aarch64_neon.rs index 1f6afac1..34fbf844 100644 --- a/src/aead/poly1305/aarch64_neon.rs +++ b/src/aead/poly1305/aarch64_neon.rs @@ -95,7 +95,9 @@ impl AeadPar4 { if self.num_cached == 0 { let group_len = segment.len().strict_sub(offset).strict_div(64).strict_mul(64); let group_end = offset.strict_add(group_len); - for group in segment[offset..group_end].chunks_exact(64) { + let (groups, remainder) = segment[offset..group_end].as_chunks::<64>(); + debug_assert!(remainder.is_empty()); + for group in groups { let (blocks, remainder) = group.as_chunks::<16>(); debug_assert!(remainder.is_empty()); assert_eq!(blocks.len(), 4, "64-byte Poly1305 group must contain four blocks"); diff --git a/src/auth/curve25519_edwards.rs b/src/auth/curve25519_edwards.rs index 4fc7eb72..c3483c1f 100644 --- a/src/auth/curve25519_edwards.rs +++ b/src/auth/curve25519_edwards.rs @@ -33,6 +33,17 @@ pub use point_avx2::{ }; /// Dispatch `[s]B` (fixed-base scalar mul) to the fastest validated CT path. +#[cfg_attr( + all( + target_arch = "x86_64", + target_os = "linux", + not(any(test, miri, feature = "portable-only")) + ), + expect( + dead_code, + reason = "x86_64 Linux library builds use the assembly fixed-base entry points" + ) +)] #[must_use] pub(crate) fn basepoint_mul_dispatch(scalar_bytes: &[u8; 32]) -> point::ExtendedPoint { #[cfg(target_arch = "x86_64")] diff --git a/src/auth/ed25519/point_avx2.rs b/src/auth/ed25519/point_avx2.rs index a15b6fa4..6feedff8 100644 --- a/src/auth/ed25519/point_avx2.rs +++ b/src/auth/ed25519/point_avx2.rs @@ -450,6 +450,13 @@ pub(crate) unsafe fn scalar_mul_vartime_avx2(point: &ExtendedPoint, scalar_bytes /// # Safety /// /// Caller must ensure AVX2 is available. +#[cfg_attr( + all(target_os = "linux", not(any(test, miri, feature = "portable-only"))), + expect( + dead_code, + reason = "x86_64 Linux library builds route fixed-base multiplication through assembly" + ) +)] #[target_feature(enable = "avx2")] pub(crate) unsafe fn scalar_mul_basepoint_avx2(scalar_bytes: &[u8; 32]) -> ExtendedPoint { use super::point::BASEPOINT_RADIX16_TABLE; @@ -867,6 +874,13 @@ pub(crate) unsafe fn scalar_mul_vartime_ifma(point: &ExtendedPoint, scalar_bytes /// # Safety /// /// Caller must ensure AVX-512 IFMA + VL are available. +#[cfg_attr( + all(target_os = "linux", not(any(test, miri, feature = "portable-only"))), + expect( + dead_code, + reason = "x86_64 Linux library builds route fixed-base multiplication through assembly" + ) +)] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] pub(crate) unsafe fn scalar_mul_basepoint_ifma(scalar_bytes: &[u8; 32]) -> ExtendedPoint { use super::point::BASEPOINT_RADIX16_TABLE; diff --git a/src/hashes/crypto/blake3/x86_64.rs b/src/hashes/crypto/blake3/x86_64.rs index c6fa176a..e7e64421 100644 --- a/src/hashes/crypto/blake3/x86_64.rs +++ b/src/hashes/crypto/blake3/x86_64.rs @@ -177,7 +177,10 @@ pub(crate) unsafe fn compress_in_place_avx2_bytes( // On ASM-supported platforms, we prefer the handwritten assembly. This intrinsics // version is kept as fallback for other x86_64 platforms (e.g., FreeBSD, illumos). -#[cfg(target_arch = "x86_64")] +#[cfg(any( + feature = "diag", + not(any(target_os = "linux", target_os = "macos", target_os = "windows")) +))] #[target_feature(enable = "avx512f,avx512vl,avx2,sse4.1,ssse3")] /// Compresses one byte-oriented block in place with AVX-512. /// @@ -202,7 +205,10 @@ pub(crate) unsafe fn compress_in_place_avx512_bytes( } } -#[cfg(target_arch = "x86_64")] +#[cfg(any( + feature = "diag", + not(any(target_os = "linux", target_os = "macos", target_os = "windows")) +))] #[target_feature(enable = "avx512f,avx512vl,avx2,sse4.1,ssse3")] /// Compresses one byte-oriented block to a chaining value with AVX-512. /// diff --git a/src/hashes/fast/xxh3/dispatch.rs b/src/hashes/fast/xxh3/dispatch.rs index e18117d7..f0690add 100644 --- a/src/hashes/fast/xxh3/dispatch.rs +++ b/src/hashes/fast/xxh3/dispatch.rs @@ -102,7 +102,7 @@ pub(crate) fn stream_accumulate_fn() -> StreamAccumulateFn { active().stream_accumulate } -#[cfg(feature = "diag")] +#[cfg(any(feature = "diag", all(test, target_arch = "x86_64")))] #[inline] #[must_use] fn kernel_id64_for_len(long_id: Xxh3KernelId, caps: Caps, len: usize) -> Xxh3KernelId { diff --git a/src/platform/detect/arch/aarch64.rs b/src/platform/detect/arch/aarch64.rs index b929e8ab..7bd534bb 100644 --- a/src/platform/detect/arch/aarch64.rs +++ b/src/platform/detect/arch/aarch64.rs @@ -96,7 +96,8 @@ fn hwcap_batch_aarch64() -> Caps { // Parse as array of (u64, u64) pairs let entries = buf.get(..n)?; - for chunk in entries.chunks_exact(16) { + let (entries, _) = entries.as_chunks::<16>(); + for chunk in entries { let a_type = u64::from_ne_bytes(chunk.get(0..8)?.try_into().ok()?); let a_val = u64::from_ne_bytes(chunk.get(8..16)?.try_into().ok()?); From 06a3e8a19cb487c1e17c87d41b33506363850563 Mon Sep 17 00:00:00 2001 From: LoadingALIAS Date: Mon, 17 Aug 2026 14:43:54 -0400 Subject: [PATCH 07/12] aead: restore s390x fixslice AES test gate and migrate ChaCha20/Poly1305 AVX2/AVX-512 kernels to as_chunks auth: gate ed25519 AVX2/IFMA point tests behind the ed25519 feature and migrate point chunking to as_chunks clippy: document missing # Safety on ct-binsec-harness AVX2/IFMA entrypoints Restoring strict cross-target and feature-matrix validation surfaced four latent CI failures: s390x test builds referenced the fixslice AES module after it was excluded from s390x test cfg, x25519-only builds pulled in ed25519-gated point_avx2 tests that were never feature-gated, and the new clippy::chunks_exact_to_as_chunks lint plus missing_safety_doc caught real gaps in the ChaCha20/Poly1305/Ed25519 SIMD kernels and the CT harness. --- src/aead/aes.rs | 4 ++++ src/aead/chacha20/x86_64_avx2.rs | 10 ++++------ src/aead/chacha20/x86_64_avx512.rs | 10 ++++------ src/aead/poly1305/x86_64_avx2_par4.rs | 9 +++------ src/auth/ed25519/point_avx2.rs | 8 +++----- tools/ct-binsec-harness/src/main.rs | 7 +++++++ 6 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/aead/aes.rs b/src/aead/aes.rs index 02f22da8..382301aa 100644 --- a/src/aead/aes.rs +++ b/src/aead/aes.rs @@ -5766,6 +5766,7 @@ mod tests { } /// FIPS 197 Appendix C.1 against the table-free fixslice AES-128 path. + #[cfg(not(target_arch = "s390x"))] #[test] fn riscv64_fixslice_matches_nist_aes128_vector() { let key: [u8; 16] = [ @@ -5784,6 +5785,7 @@ mod tests { assert_eq!(block, expected); } + #[cfg(not(target_arch = "s390x"))] #[test] fn riscv64_fixslice_128_4blocks_matches_portable() { let key = [0xC4u8; KEY_SIZE_128]; @@ -5806,6 +5808,7 @@ mod tests { assert_eq!(blocks, expected); } + #[cfg(not(target_arch = "s390x"))] #[test] fn riscv64_fixslice_matches_nist_aes256_vector() { let key: [u8; 32] = [ @@ -5825,6 +5828,7 @@ mod tests { assert_eq!(block, expected); } + #[cfg(not(target_arch = "s390x"))] #[test] fn riscv64_fixslice_4blocks_matches_portable() { let key = [0x3cu8; KEY_SIZE]; diff --git a/src/aead/chacha20/x86_64_avx2.rs b/src/aead/chacha20/x86_64_avx2.rs index 6ee8a08b..a6752bfe 100644 --- a/src/aead/chacha20/x86_64_avx2.rs +++ b/src/aead/chacha20/x86_64_avx2.rs @@ -44,8 +44,8 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: ); let mut counter = initial_counter; - let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); - for chunk in &mut batches { + let (batches, remainder) = buffer.as_chunks_mut::<{ BLOCK_SIZE * BLOCKS_PER_BATCH }>(); + for chunk in batches { debug_assert!(counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); let mut x0 = _mm256_set1_epi32(0x6170_7865u32.cast_signed()); @@ -183,16 +183,14 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: counter = counter.wrapping_add(COUNTERS_PER_BATCH); } - let remainder = batches.into_remainder(); - let mut x4_batches = remainder.chunks_exact_mut(BLOCK_SIZE * x86_ssse3_x4::BLOCKS_PER_BATCH); - for chunk in &mut x4_batches { + let (x4_batches, remainder) = remainder.as_chunks_mut::<{ BLOCK_SIZE * x86_ssse3_x4::BLOCKS_PER_BATCH }>(); + for chunk in x4_batches { // SAFETY: AVX2-capable CPUs provide the SSSE3 instructions used by the // 4-block tail kernel, and `chunk` is exactly 4 ChaCha20 blocks. unsafe { x86_ssse3_x4::xor_blocks(key, counter, nonce, chunk) }; counter = counter.wrapping_add(x86_ssse3_x4::COUNTERS_PER_BATCH); } - let remainder = x4_batches.into_remainder(); if !remainder.is_empty() { xor_keystream_portable(key, counter, nonce, remainder); } diff --git a/src/aead/chacha20/x86_64_avx512.rs b/src/aead/chacha20/x86_64_avx512.rs index 6635f62f..5fc5dbd2 100644 --- a/src/aead/chacha20/x86_64_avx512.rs +++ b/src/aead/chacha20/x86_64_avx512.rs @@ -36,8 +36,8 @@ pub(super) unsafe fn xor_keystream( #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq")] unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { let mut counter = initial_counter; - let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); - for chunk in &mut batches { + let (batches, remainder) = buffer.as_chunks_mut::<{ BLOCK_SIZE * BLOCKS_PER_BATCH }>(); + for chunk in batches { debug_assert!(counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); let mut x0 = _mm512_set1_epi32(0x6170_7865u32.cast_signed()); @@ -236,16 +236,14 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: counter = counter.wrapping_add(COUNTERS_PER_BATCH); } - let remainder = batches.into_remainder(); - let mut x4_batches = remainder.chunks_exact_mut(BLOCK_SIZE * x86_ssse3_x4::BLOCKS_PER_BATCH); - for chunk in &mut x4_batches { + let (x4_batches, remainder) = remainder.as_chunks_mut::<{ BLOCK_SIZE * x86_ssse3_x4::BLOCKS_PER_BATCH }>(); + for chunk in x4_batches { // SAFETY: AVX-512-ready CPUs provide the SSSE3 instructions used by the // 4-block tail kernel, and `chunk` is exactly 4 ChaCha20 blocks. unsafe { x86_ssse3_x4::xor_blocks(key, counter, nonce, chunk) }; counter = counter.wrapping_add(x86_ssse3_x4::COUNTERS_PER_BATCH); } - let remainder = x4_batches.into_remainder(); if !remainder.is_empty() { xor_keystream_portable(key, counter, nonce, remainder); } diff --git a/src/aead/poly1305/x86_64_avx2_par4.rs b/src/aead/poly1305/x86_64_avx2_par4.rs index 048d4722..3efd0a1e 100644 --- a/src/aead/poly1305/x86_64_avx2_par4.rs +++ b/src/aead/poly1305/x86_64_avx2_par4.rs @@ -703,15 +703,12 @@ pub(super) unsafe fn authenticate_aead_par4( // Reimplements padded-segment logic from `update_padded_segment` for 4-way batching. for segment in [aad, ciphertext] { - let mut chunks = segment.chunks_exact(16); - for chunk in &mut chunks { - let mut block = [0u8; 16]; - block.copy_from_slice(chunk); + let (chunks, rem) = segment.as_chunks::<16>(); + for chunk in chunks { // SAFETY: the caller guarantees AVX2. The loop produces consecutive full blocks; `push_block` preserves // `num_cached < 4` and keeps the cache, powers, and accumulator in the same key stream. - num_cached = unsafe { push_block(block, &mut cached, num_cached, &mut acc, r1, r2) }; + num_cached = unsafe { push_block(*chunk, &mut cached, num_cached, &mut acc, r1, r2) }; } - let rem = chunks.remainder(); if !rem.is_empty() { let mut block = [0u8; 16]; block[..rem.len()].copy_from_slice(rem); diff --git a/src/auth/ed25519/point_avx2.rs b/src/auth/ed25519/point_avx2.rs index 6feedff8..a138caf7 100644 --- a/src/auth/ed25519/point_avx2.rs +++ b/src/auth/ed25519/point_avx2.rs @@ -490,7 +490,7 @@ pub unsafe fn diag_select_basepoint_cached_avx2_limb_digest(digit: i8) -> [u64; let selected = select_signed_cached_avx2(&BASEPOINT_RADIX16_TABLE[0], digit, &affine_k, &identity); let fields = selected.0.split(); let mut out = [0u64; 20]; - for (chunk, field) in out.chunks_exact_mut(5).zip(fields.iter()) { + for (chunk, field) in out.as_chunks_mut::<5>().0.iter_mut().zip(fields.iter()) { chunk.copy_from_slice(field.limbs()); } out @@ -914,7 +914,7 @@ pub unsafe fn diag_select_basepoint_cached_ifma_limb_digest(digit: i8) -> [u64; let selected = select_signed_cached_ifma(&BASEPOINT_RADIX16_TABLE[0], digit, &affine_k, &identity); let fields = selected.0.split(); let mut out = [0u64; 20]; - for (chunk, field) in out.chunks_exact_mut(5).zip(fields.iter()) { + for (chunk, field) in out.as_chunks_mut::<5>().0.iter_mut().zip(fields.iter()) { chunk.copy_from_slice(field.limbs()); } out @@ -1085,6 +1085,7 @@ pub(crate) unsafe fn straus_wnaf_vartime_ifma(s: &[u8; 32], h: &[u8; 32], a: &Ex #[cfg(test)] #[cfg(target_arch = "x86_64")] +#[cfg(feature = "ed25519")] mod tests { use super::{ExtendedPoint, *}; @@ -1320,7 +1321,6 @@ mod tests { } } - #[cfg(feature = "ed25519")] #[test] fn scalar_mul_basepoint_rfc8032_vector1() { if !std::arch::is_x86_feature_detected!("avx2") { @@ -1345,7 +1345,6 @@ mod tests { } } - #[cfg(feature = "ed25519")] #[test] fn straus_matches_scalar() { if !std::arch::is_x86_feature_detected!("avx2") { @@ -1371,7 +1370,6 @@ mod tests { } } - #[cfg(feature = "ed25519")] #[test] fn straus_matches_scalar_large_scalars() { if !std::arch::is_x86_feature_detected!("avx2") { diff --git a/tools/ct-binsec-harness/src/main.rs b/tools/ct-binsec-harness/src/main.rs index 3499d8a5..58acfba2 100644 --- a/tools/ct-binsec-harness/src/main.rs +++ b/tools/ct-binsec-harness/src/main.rs @@ -885,6 +885,9 @@ pub extern "C" fn ct_binsec_rsa_private_component_validation_32() -> ! { #[unsafe(no_mangle)] #[inline(never)] #[target_feature(enable = "avx2")] +/// # Safety +/// +/// The caller must ensure AVX2 is available before invoking this entrypoint. pub unsafe extern "C" fn ct_binsec_ed25519_select_basepoint_cached_avx2() -> ! { // SAFETY: This pointer references a fixed harness global with static storage. let digit = unsafe { ptr::read_volatile(ptr::addr_of!(CT_BINSEC_ED25519_DIGIT)) }; @@ -905,6 +908,10 @@ pub unsafe extern "C" fn ct_binsec_ed25519_select_basepoint_cached_avx2() -> ! { #[unsafe(no_mangle)] #[inline(never)] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] +/// # Safety +/// +/// The caller must ensure AVX2, AVX-512 IFMA, and AVX-512 VL are available +/// before invoking this entrypoint. pub unsafe extern "C" fn ct_binsec_ed25519_select_basepoint_cached_ifma() -> ! { // SAFETY: This pointer references a fixed harness global with static storage. let digit = unsafe { ptr::read_volatile(ptr::addr_of!(CT_BINSEC_ED25519_DIGIT)) }; From d0c7ff3e653dbc62bee1a534da3373c7060e4593 Mon Sep 17 00:00:00 2001 From: LoadingALIAS Date: Mon, 17 Aug 2026 19:56:39 -0400 Subject: [PATCH 08/12] auth: tighten ed25519-only AVX2/IFMA vartime internals to feature=ed25519 The previous ed25519 test-feature gate on point_avx2's mod tests exposed a second layer of latent breakage under --features x25519 (no ed25519): several AVX2/IFMA internals (wNAF tables, cached-point builders, field squaring/shuffle helpers in point_avx2, field_avx2, and field_ifma) were gated any(test, ed25519) so they kept compiling under test alone, but their only real callers are the vartime scalar-mul and Straus verification paths, which are ed25519-only and were never reachable without it. That left them dead code once the test module required ed25519 too. Tighten those internals to feature = "ed25519" and gate the field_avx2 and field_ifma test modules the same way as point_avx2's. Traced the call graph to confirm none of this is reachable from x25519's fixed-base basepoint_mul_dispatch path, which stays available under x25519-only. Verified via a x86_64 cross-compile (zig cc) since ring/aws-lc-sys dev dependencies can't cross-link natively here: x25519-only lib+tests check and clippy are clean, ed25519+x25519 combined check is clean, and the native all-features suite still passes at 1061/1061. --- src/auth/ed25519/field_avx2.rs | 45 +++++++++++++++++----------------- src/auth/ed25519/field_ifma.rs | 33 +++++++++++++------------ src/auth/ed25519/point_avx2.rs | 32 ++++++++++++------------ 3 files changed, 56 insertions(+), 54 deletions(-) diff --git a/src/auth/ed25519/field_avx2.rs b/src/auth/ed25519/field_avx2.rs index 69a8ddad..7c64d58a 100644 --- a/src/auth/ed25519/field_avx2.rs +++ b/src/auth/ed25519/field_avx2.rs @@ -55,23 +55,23 @@ pub(crate) enum Shuffle { /// Swap right pair only: (A, B, C, D) → (A, B, D, C) SwapCD, /// Broadcast A: (A, B, C, D) → (A, A, A, A) - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] BroadcastA, /// Broadcast B: (A, B, C, D) → (B, B, B, B) - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] BroadcastB, /// (A, B, C, D) → (C, A, C, A) - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] AlternateCA, /// (A, B, C, D) → (D, B, B, D) - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] OuterDInnerB, /// (A, B, C, D) → (A, D, D, A) OuterAInnerD, /// (A, B, C, D) → (C, B, C, B) AlternateCB, /// (A, B, C, D) → (A, B, A, B) - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] RepeatAB, } @@ -83,17 +83,17 @@ impl Shuffle { Self::SwapPairs => [1, 0, 3, 2, 5, 4, 7, 6], Self::SwapAB => [1, 0, 3, 2, 4, 5, 6, 7], Self::SwapCD => [0, 1, 2, 3, 5, 4, 7, 6], - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Self::BroadcastA => [0, 0, 2, 2, 0, 0, 2, 2], - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Self::BroadcastB => [1, 1, 3, 3, 1, 1, 3, 3], - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Self::AlternateCA => [4, 0, 6, 2, 4, 0, 6, 2], - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Self::OuterDInnerB => [5, 1, 7, 3, 1, 5, 3, 7], Self::OuterAInnerD => [0, 5, 2, 7, 5, 0, 7, 2], Self::AlternateCB => [4, 1, 6, 3, 4, 1, 6, 3], - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Self::RepeatAB => [0, 1, 2, 3, 0, 1, 2, 3], } } @@ -109,7 +109,7 @@ impl Shuffle { #[repr(u8)] pub(crate) enum Lanes { /// Select C lanes: positions 4, 6 - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] C = 0b0101_0000, /// Select D lanes: positions 5, 7 D = 0b1010_0000, @@ -118,13 +118,13 @@ pub(crate) enum Lanes { /// Select A and C lanes: positions 0, 2, 4, 6 AC = 0b0101_0101, /// Select A and D lanes: positions 0, 2, 5, 7 - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] AD = 0b1010_0101, /// Select B and C lanes: positions 1, 3, 4, 6 - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] BC = 0b0101_1010, /// Select every lane except A: positions 1, 3, 4, 5, 6, 7 - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] ExceptA = 0b1111_1010, } @@ -426,16 +426,16 @@ impl FieldElement2625x4 { } match lanes { - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Lanes::C => do_blend!(0b0101_0000), Lanes::D => do_blend!(0b1010_0000), Lanes::AB => do_blend!(0b0000_1111), Lanes::AC => do_blend!(0b0101_0101), - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Lanes::AD => do_blend!(0b1010_0101), - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Lanes::BC => do_blend!(0b0101_1010), - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Lanes::ExceptA => do_blend!(0b1111_1010), } } @@ -784,7 +784,7 @@ impl FieldElement2625x4 { /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] fn square_accum(&self) -> [__m256i; 10] { let v19 = _mm256_set1_epi64x(19); @@ -908,7 +908,7 @@ impl FieldElement2625x4 { /// # Safety /// /// Calls from outside an AVX2-enabled context require runtime AVX2 support. - #[cfg(test)] + #[cfg(all(test, feature = "ed25519"))] #[inline] #[target_feature(enable = "avx2")] pub(crate) fn square(&self) -> Self { @@ -933,7 +933,7 @@ impl FieldElement2625x4 { /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] pub(crate) fn square_and_negate_d(&self) -> Self { let mut z = self.square_accum(); Self::negate_d_accum(&mut z); @@ -953,7 +953,7 @@ impl FieldElement2625x4 { /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] fn negate_d_accum(z: &mut [__m256i; 10]) { // p × 2^37 per limb (radix-26/25): let bias_even_0 = _mm256_set1_epi64x(((1i64 << 26) - 19) << 37); @@ -987,6 +987,7 @@ impl FieldElement2625x4 { #[cfg(test)] #[cfg(target_arch = "x86_64")] +#[cfg(feature = "ed25519")] mod tests { use super::{FieldElement, *}; diff --git a/src/auth/ed25519/field_ifma.rs b/src/auth/ed25519/field_ifma.rs index ab47b915..923cdf58 100644 --- a/src/auth/ed25519/field_ifma.rs +++ b/src/auth/ed25519/field_ifma.rs @@ -30,7 +30,7 @@ use super::{ // Constants const MASK51: i64 = (1i64 << 51) - 1; -#[cfg(test)] +#[cfg(all(test, feature = "ed25519"))] const MASK52: i64 = (1i64 << 52) - 1; /// Subtraction bias: 2p in radix-51. Limb 0 accounts for the -19 term. @@ -96,7 +96,7 @@ fn madd52hi(acc: __m256i, a: __m256i, b: __m256i) -> __m256i { /// # Safety /// /// Caller must ensure AVX2 is available. -#[cfg(test)] +#[cfg(all(test, feature = "ed25519"))] #[inline] #[target_feature(enable = "avx2")] fn select_by_bit(bit: __m256i, val: __m256i) -> __m256i { @@ -284,17 +284,17 @@ impl FieldElement51x4 { Shuffle::SwapPairs => do_shuffle!(0b10_11_00_01), Shuffle::SwapAB => do_shuffle!(0b11_10_00_01), Shuffle::SwapCD => do_shuffle!(0b10_11_01_00), - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Shuffle::BroadcastA => do_shuffle!(0b00_00_00_00), - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Shuffle::BroadcastB => do_shuffle!(0b01_01_01_01), - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Shuffle::AlternateCA => do_shuffle!(0b00_10_00_10), - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Shuffle::OuterDInnerB => do_shuffle!(0b11_01_01_11), Shuffle::OuterAInnerD => do_shuffle!(0b00_11_11_00), Shuffle::AlternateCB => do_shuffle!(0b01_10_01_10), - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Shuffle::RepeatAB => do_shuffle!(0b01_00_01_00), } } @@ -324,16 +324,16 @@ impl FieldElement51x4 { } match lanes { - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Lanes::C => do_blend!(0b0011_0000), Lanes::D => do_blend!(0b1100_0000), Lanes::AB => do_blend!(0b0000_1111), Lanes::AC => do_blend!(0b0011_0011), - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Lanes::AD => do_blend!(0b1100_0011), - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Lanes::BC => do_blend!(0b0011_1100), - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] Lanes::ExceptA => do_blend!(0b1111_1100), } } @@ -596,7 +596,7 @@ impl FieldElement51x4 { /// # Safety /// /// Caller must ensure AVX-512 IFMA + VL are available. - #[cfg(test)] + #[cfg(all(test, feature = "ed25519"))] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] pub(crate) fn mul_unreduced(&self, rhs: &Self) -> Self { let zero = _mm256_setzero_si256(); @@ -898,7 +898,7 @@ impl FieldElement51x4 { /// # Safety /// /// Caller must ensure AVX-512 IFMA + VL are available. - #[cfg(test)] + #[cfg(all(test, feature = "ed25519"))] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] pub(crate) fn mul_small_unreduced(&self, small: &Self) -> Self { let zero = _mm256_setzero_si256(); @@ -967,7 +967,7 @@ impl FieldElement51x4 { /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] pub(crate) fn square(&self) -> Self { let zero = _mm256_setzero_si256(); let f = &self.0; @@ -1071,7 +1071,7 @@ impl FieldElement51x4 { /// # Safety /// /// Caller must ensure AVX-512 IFMA + VL are available. - #[cfg(test)] + #[cfg(all(test, feature = "ed25519"))] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] fn square_wide_fold(&self) -> [__m256i; 5] { let zero = _mm256_setzero_si256(); @@ -1192,7 +1192,7 @@ impl FieldElement51x4 { /// # Safety /// /// Caller must ensure AVX-512 IFMA + VL are available. - #[cfg(test)] + #[cfg(all(test, feature = "ed25519"))] #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] pub(crate) fn square_and_negate_d_wide(&self) -> Self { @@ -1283,6 +1283,7 @@ impl FieldElement51x4 { #[cfg(test)] #[cfg(target_arch = "x86_64")] +#[cfg(feature = "ed25519")] mod tests { use super::{FieldElement, *}; diff --git a/src/auth/ed25519/point_avx2.rs b/src/auth/ed25519/point_avx2.rs index a138caf7..babe4055 100644 --- a/src/auth/ed25519/point_avx2.rs +++ b/src/auth/ed25519/point_avx2.rs @@ -12,7 +12,7 @@ //! scaling all output coordinates by `d2²` — which cancels in projective //! coordinates. -#[cfg(all(target_arch = "x86_64", any(test, feature = "ed25519")))] +#[cfg(all(target_arch = "x86_64", feature = "ed25519"))] use core::arch::x86_64::_mm256_loadu_si256; #[cfg(all(target_arch = "x86_64", feature = "ed25519"))] @@ -27,7 +27,7 @@ use super::{ }; #[cfg(target_arch = "x86_64")] #[path = "basepoint_table_ifma.rs"] -#[cfg(any(test, feature = "ed25519"))] +#[cfg(feature = "ed25519")] mod basepoint_table_ifma; /// Hamburg constants for the curve `d = -d1/d2`. @@ -142,7 +142,7 @@ impl ExtendedPointAvx2 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] pub(crate) fn double(&self) -> Self { // Step 1: Build (X, Y, Z, X+Y) for squaring. let ab = self.0.shuffle(Shuffle::RepeatAB); // (X, Y, X, Y) @@ -304,7 +304,7 @@ fn hamburg_affine_constants() -> FieldElement2625x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[cfg(any(test, feature = "ed25519"))] +#[cfg(feature = "ed25519")] fn cached_from_affine(cp: &CachedPoint, constants: &FieldElement2625x4) -> CachedPointAvx2 { let (y_plus_x, y_minus_x, t2d) = cp.components(); let packed = FieldElement2625x4::new(y_minus_x, y_plus_x, &FieldElement::ONE, t2d); @@ -348,7 +348,7 @@ fn select_signed_cached_avx2( /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[cfg(any(test, feature = "ed25519"))] +#[cfg(feature = "ed25519")] fn add_wnaf_digit_cached_avx2( acc: ExtendedPointAvx2, table: &[CachedPoint; 8], @@ -375,7 +375,7 @@ fn add_wnaf_digit_cached_avx2( /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[cfg(any(test, feature = "ed25519"))] +#[cfg(feature = "ed25519")] fn add_signed_runtime_cached_avx2( acc: ExtendedPointAvx2, table: &[CachedPointAvx2; 8], @@ -399,7 +399,7 @@ fn add_signed_runtime_cached_avx2( /// /// Caller must ensure AVX2 is available. #[target_feature(enable = "avx2")] -#[cfg(any(test, feature = "ed25519"))] +#[cfg(feature = "ed25519")] fn cached_multiples_avx2(point: &ExtendedPointAvx2) -> [CachedPointAvx2; 8] { let mut acc = *point; let point_cached = point.to_cached(); @@ -505,7 +505,7 @@ pub unsafe fn diag_select_basepoint_cached_avx2_limb_digest(digit: i8) -> [u64; /// /// Caller must ensure AVX2 is available. #[target_feature(enable = "avx2")] -#[cfg(any(test, feature = "ed25519"))] +#[cfg(feature = "ed25519")] fn odd_multiples_avx2(point: &ExtendedPointAvx2) -> [CachedPointAvx2; N] { let p2 = point.double(); let p2_cached = p2.to_cached(); @@ -527,7 +527,7 @@ fn odd_multiples_avx2(point: &ExtendedPointAvx2) -> [CachedPoint /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[cfg(any(test, feature = "ed25519"))] +#[cfg(feature = "ed25519")] fn add_wnaf_digit_avx2(acc: ExtendedPointAvx2, table: &[CachedPointAvx2], digit: i8) -> ExtendedPointAvx2 { let index = usize::from((digit.unsigned_abs().wrapping_sub(1)) / 2); let Some(point) = table.get(index) else { @@ -683,7 +683,7 @@ impl ExtendedPointIfma { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[cfg(any(test, feature = "ed25519"))] + #[cfg(feature = "ed25519")] pub(crate) fn double(&self) -> Self { // Prepare (X, Y, Z, X+Y) for squaring. let tmp0 = self.0.shuffle(Shuffle::SwapPairs); // (Y, X, _, _) @@ -811,7 +811,7 @@ fn select_signed_cached_ifma( /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[cfg(any(test, feature = "ed25519"))] +#[cfg(feature = "ed25519")] fn add_signed_runtime_cached_ifma( acc: ExtendedPointIfma, table: &[CachedPointIfma; 8], @@ -834,7 +834,7 @@ fn add_signed_runtime_cached_ifma( /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[cfg(any(test, feature = "ed25519"))] +#[cfg(feature = "ed25519")] fn cached_multiples_ifma(point: &ExtendedPointIfma) -> [CachedPointIfma; 8] { let mut acc = *point; let point_cached = point.to_cached(); @@ -936,7 +936,7 @@ pub unsafe fn diag_select_basepoint_cached_ifma_limb_digest(digit: i8) -> [u64; /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[cfg(any(test, feature = "ed25519"))] +#[cfg(feature = "ed25519")] fn odd_multiples_ifma(point: &ExtendedPointIfma) -> [CachedPointIfma; N] { let p2 = point.double(); let p2_cached = p2.to_cached(); @@ -961,7 +961,7 @@ fn odd_multiples_ifma(point: &ExtendedPointIfma) -> [CachedPoint /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[cfg(any(test, feature = "ed25519"))] +#[cfg(feature = "ed25519")] fn add_wnaf_digit_ifma(acc: ExtendedPointIfma, table: &[CachedPointIfma], digit: i8) -> ExtendedPointIfma { let index = usize::from((digit.unsigned_abs().wrapping_sub(1)) / 2); let Some(point) = table.get(index) else { @@ -982,7 +982,7 @@ fn add_wnaf_digit_ifma(acc: ExtendedPointIfma, table: &[CachedPointIfma], digit: /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[cfg(any(test, feature = "ed25519"))] +#[cfg(feature = "ed25519")] fn load_cached_ifma_raw(entry: &[[i64; 4]; 5]) -> CachedPointIfma { // SAFETY: AVX2 is active in this function, and every inner array provides 32 initialized bytes for an unaligned load. let limbs = unsafe { @@ -1004,7 +1004,7 @@ fn load_cached_ifma_raw(entry: &[[i64; 4]; 5]) -> CachedPointIfma { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[cfg(any(test, feature = "ed25519"))] +#[cfg(feature = "ed25519")] fn add_wnaf_digit_ifma_raw(acc: ExtendedPointIfma, table: &[[[i64; 4]; 5]], digit: i8) -> ExtendedPointIfma { let index = usize::from((digit.unsigned_abs().wrapping_sub(1)) / 2); let Some(entry) = table.get(index) else { From 906a43736f59d1bd52ae44804eeef8a8f4431a3f Mon Sep 17 00:00:00 2001 From: LoadingALIAS Date: Mon, 17 Aug 2026 20:57:38 -0400 Subject: [PATCH 09/12] aead: gate VAES-512 encrypt_4blocks behind aes-gcm/aes-gcm-siv auth: keep basepoint_mul_dispatch reachable under single-curve feature sets Two more dead-code gaps surfaced by the stricter feature-matrix check. x86_64_ni's plain encrypt_4blocks/encrypt_4blocks_128 were the only ungated variants in that file (their _aesni and 16-block siblings already carry the aes-gcm/aes-gcm-siv gate); under aegis256-only they had no caller left. basepoint_mul_dispatch only had test-reachable callers via ed25519's and x25519's own assembly-routing dispatch, and ed25519's path lacks x25519's test escape hatch, so under ed25519-only ("signatures") it went dead too; added direct differential tests instead of touching that dispatch structure. Verified via x86_64 cross-compile (zig cc): all 58 feature-matrix profiles in scripts/lib/feature-profiles.sh now pass cargo check --lib --tests. --- src/aead/aes/x86_64_ni.rs | 2 ++ src/auth/curve25519_edwards.rs | 44 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/aead/aes/x86_64_ni.rs b/src/aead/aes/x86_64_ni.rs index d2d697e7..c9fbb60c 100644 --- a/src/aead/aes/x86_64_ni.rs +++ b/src/aead/aes/x86_64_ni.rs @@ -90,6 +90,7 @@ pub(super) unsafe fn expand_key(key: &[u8; 32]) -> NiRoundKeys { /// /// # Safety /// Caller must ensure AVX-512F + AVX-512VL + VAES + AES + SSE2. +#[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] #[target_feature(enable = "aes,sse2,avx512f,avx512vl,vaes")] #[inline] pub(super) unsafe fn encrypt_4blocks(keys: &NiRoundKeys, blocks: __m512i) -> __m512i { @@ -324,6 +325,7 @@ pub(super) unsafe fn expand_key_128(key: &[u8; 16]) -> Ni128RoundKeys { /// /// # Safety /// Caller must ensure AVX-512F + AVX-512VL + VAES + AES + SSE2. +#[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] #[target_feature(enable = "aes,sse2,avx512f,avx512vl,vaes")] #[inline] pub(super) unsafe fn encrypt_4blocks_128(keys: &Ni128RoundKeys, blocks: __m512i) -> __m512i { diff --git a/src/auth/curve25519_edwards.rs b/src/auth/curve25519_edwards.rs index c3483c1f..85e13729 100644 --- a/src/auth/curve25519_edwards.rs +++ b/src/auth/curve25519_edwards.rs @@ -87,3 +87,47 @@ fn scalar_radix_16(bytes: &[u8; 32]) -> [i8; 64] { digits } + +#[cfg(test)] +mod tests { + use super::*; + + // Keeps `basepoint_mul_dispatch` reachable under every feature + // combination: the x86_64/aarch64 assembly entry points route around it + // in ed25519's and x25519's own callers, so without a direct test caller + // it goes dead under feature sets that enable only one of the two curves. + // Encoding is curve-specific (`to_bytes` needs ed25519, `to_montgomery_u` + // needs x25519), so each active curve gets its own comparison. + + #[cfg(feature = "ed25519")] + #[test] + fn basepoint_mul_dispatch_matches_portable_edwards_encoding() { + let mut scalar = [0u8; 32]; + scalar[0] = 9; + + let dispatched = basepoint_mul_dispatch(&scalar); + let portable = point::ExtendedPoint::scalar_mul_basepoint(&scalar); + + assert_eq!( + dispatched.to_bytes(), + portable.to_bytes(), + "dispatched basepoint mul should match the portable reference" + ); + } + + #[cfg(feature = "x25519")] + #[test] + fn basepoint_mul_dispatch_matches_portable_montgomery_encoding() { + let mut scalar = [0u8; 32]; + scalar[0] = 9; + + let dispatched = basepoint_mul_dispatch(&scalar); + let portable = point::ExtendedPoint::scalar_mul_basepoint(&scalar); + + assert_eq!( + dispatched.to_montgomery_u().normalize().to_bytes(), + portable.to_montgomery_u().normalize().to_bytes(), + "dispatched basepoint mul should match the portable reference" + ); + } +} From 52b30318bba6a66ed01a5ea8c17ea87e157511af Mon Sep 17 00:00:00 2001 From: LoadingALIAS Date: Tue, 18 Aug 2026 14:17:17 -0400 Subject: [PATCH 10/12] auth: gate RSA x86-64 montgomery evidence test on observable BMI2/ADX clippy: fold ct-binsec-harness AVX2/IFMA digests without truncating casts x86-64 BMI2/ADX discovery is std-only, so under --no-default-features --features full caps() reports only the compile-time feature set, the assembly backend is unreachable, and the differential test's hard assert fired at profile 1 of the executable feature matrix. Gate the test and the four Montgomery helpers sharing its predicate on the condition that makes the backend observable, and keep the assertion hard inside it. The two x86_64 BINSEC entrypoints folded their u64 accumulator with an `as u8` cast while their six portable siblings use to_le_bytes()[0]. cast_possible_truncation is deny-level in the independent-workspace lint, so Quality failed on x86_64 only. --- src/auth/rsa.rs | 32 ++++++++++++++++++++++++----- tools/ct-binsec-harness/src/main.rs | 10 ++++----- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/auth/rsa.rs b/src/auth/rsa.rs index b3214b18..248f5059 100644 --- a/src/auth/rsa.rs +++ b/src/auth/rsa.rs @@ -14026,7 +14026,11 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[cfg(all( any( all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")), - all(target_arch = "x86_64", target_os = "linux") + all( + target_arch = "x86_64", + target_os = "linux", + any(feature = "std", all(target_feature = "bmi2", target_feature = "adx")) + ) ), not(feature = "portable-only"), not(miri) @@ -14046,7 +14050,11 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[cfg(all( any( all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")), - all(target_arch = "x86_64", target_os = "linux") + all( + target_arch = "x86_64", + target_os = "linux", + any(feature = "std", all(target_feature = "bmi2", target_feature = "adx")) + ) ), not(feature = "portable-only"), not(miri) @@ -14068,7 +14076,11 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[cfg(all( any( all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")), - all(target_arch = "x86_64", target_os = "linux") + all( + target_arch = "x86_64", + target_os = "linux", + any(feature = "std", all(target_feature = "bmi2", target_feature = "adx")) + ) ), not(feature = "portable-only"), not(miri) @@ -14084,7 +14096,11 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[cfg(all( any( all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")), - all(target_arch = "x86_64", target_os = "linux") + all( + target_arch = "x86_64", + target_os = "linux", + any(feature = "std", all(target_feature = "bmi2", target_feature = "adx")) + ) ), not(feature = "portable-only"), not(miri) @@ -14199,11 +14215,17 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert_aarch64_rsa_montgomery_backend_matches_portable!(rsa_aarch64_asm); } + // x86-64 BMI2/ADX discovery is `std`-only: without it `caps()` reports just + // the compile-time feature set, so the assembly backend is unreachable and + // there is no second implementation to differentially test. Compile this + // evidence test exactly when the backend can be observed as available, and + // keep the assertion hard inside it. #[cfg(all( target_arch = "x86_64", target_os = "linux", not(feature = "portable-only"), - not(miri) + not(miri), + any(feature = "std", all(target_feature = "bmi2", target_feature = "adx")) ))] #[test] fn x86_64_linux_rsa_montgomery_asm_matches_portable_across_supported_widths() { diff --git a/tools/ct-binsec-harness/src/main.rs b/tools/ct-binsec-harness/src/main.rs index 58acfba2..c6ac58c1 100644 --- a/tools/ct-binsec-harness/src/main.rs +++ b/tools/ct-binsec-harness/src/main.rs @@ -899,9 +899,8 @@ pub unsafe extern "C" fn ct_binsec_ed25519_select_basepoint_cached_avx2() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done( - (acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8, - ) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[cfg(target_arch = "x86_64")] @@ -923,9 +922,8 @@ pub unsafe extern "C" fn ct_binsec_ed25519_select_basepoint_cached_ifma() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done( - (acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8, - ) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] From 41a6b25a9ad306d8b585b1bc7176d1510a9476d3 Mon Sep 17 00:00:00 2001 From: LoadingALIAS Date: Tue, 18 Aug 2026 14:56:43 -0400 Subject: [PATCH 11/12] aead: gate AES ECB batch helpers and VAES-512 GCM-SIV externs to their live callers fuzz-packages/surface-hex-parse pins rscrypto with aes-gcm but without aes-gcm-siv, and no declared feature profile covers that combination, so the independent-workspace lint was the only lane that saw it. On x86_64 that build left aes128/aes256_encrypt_blocks_ecb and both rscrypto_aes*_gcmsiv_ctr_16x_vaes512_x86_64_linux declarations without a caller, and dead_code is deny-level. Gate the two extern declarations on aes-gcm-siv, which is the only feature whose code path calls them. Gate the two ECB batch helpers on the union of their live callers: GCM-SIV key derivation on any arch, the batch CTR path on the arches carrying a block-batch kernel, and cfg(test) -- mod aes is also compiled under all(feature = "aegis256", test), where the two unit tests are what keep the aarch64 kernels reachable. --- src/aead/aes.rs | 32 ++++++++++++++++++++++++++++++++ src/aead/aes/x86_64/asm.rs | 2 ++ 2 files changed, 34 insertions(+) diff --git a/src/aead/aes.rs b/src/aead/aes.rs index 382301aa..1d8c792e 100644 --- a/src/aead/aes.rs +++ b/src/aead/aes.rs @@ -1764,6 +1764,22 @@ pub(crate) fn aes128_encrypt_block(ek: &Aes128EncKey, block: &mut [u8; BLOCK_SIZ /// instruction or the RV64 4-block kernels when available, otherwise calls /// the per-block dispatcher. Used by `riscv64` from the AES-128 CTR paths /// and by AES-128-GCM-SIV key derivation. +// Live callers are GCM-SIV key derivation (any arch), the batch CTR path on the +// arches that have a block-batch kernel, and the unit tests below -- `mod aes` +// is also compiled for `aegis256` under `cfg(test)`. +#[cfg(any( + test, + feature = "aes-gcm-siv", + all( + feature = "aes-gcm", + any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x" + ) + ) +))] #[inline] pub(crate) fn aes128_encrypt_blocks_ecb(ek: &Aes128EncKey, blocks: &mut [[u8; BLOCK_SIZE]]) { #[cfg(target_arch = "aarch64")] @@ -1925,6 +1941,22 @@ fn aes128_encrypt_block_portable(rk: &[u32; EXPANDED_KEY_WORDS_128], block: &mut /// On s390x this issues a single KM instruction for all `blocks`, /// avoiding per-block parameter-block setup overhead. On other platforms /// falls back to per-block dispatch. +// Live callers are GCM-SIV key derivation (any arch), the batch CTR path on the +// arches that have a block-batch kernel, and the unit tests below -- `mod aes` +// is also compiled for `aegis256` under `cfg(test)`. +#[cfg(any( + test, + feature = "aes-gcm-siv", + all( + feature = "aes-gcm", + any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x" + ) + ) +))] #[inline] pub(crate) fn aes256_encrypt_blocks_ecb(ek: &Aes256EncKey, blocks: &mut [[u8; BLOCK_SIZE]]) { #[cfg(target_arch = "aarch64")] diff --git a/src/aead/aes/x86_64/asm.rs b/src/aead/aes/x86_64/asm.rs index c266ed69..24a09024 100644 --- a/src/aead/aes/x86_64/asm.rs +++ b/src/aead/aes/x86_64/asm.rs @@ -140,6 +140,7 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); + #[cfg(feature = "aes-gcm-siv")] pub(super) fn rscrypto_aes128_gcmsiv_ctr_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -147,6 +148,7 @@ unsafe extern "C" { len: usize, ) -> usize; + #[cfg(feature = "aes-gcm-siv")] pub(super) fn rscrypto_aes256_gcmsiv_ctr_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, From 61b95d30daf130c7081dc931b1e721b041665a92 Mon Sep 17 00:00:00 2001 From: LoadingALIAS Date: Tue, 18 Aug 2026 22:52:18 -0400 Subject: [PATCH 12/12] benchmarks: refresh Linux scorecard from the 2026-08-18 eight-runner CI pass bench: anchor perf chart bars at 1.00x parity and scale the axis from data docs: publish the 2026-08-18 aggregate and lift the HMAC equivalence caveat Extract run #32185659553 at commit 7eb44e9 into benchmark_results/2026-08-18 and regenerate every OVERVIEW section from it. The RISE RISC-V runner did not execute, so all aggregates cover eight platforms rather than nine. Linux fastest-external geomean moves 1.59x to 1.62x and checksums 5.18x to 6.18x. The RustCrypto HMAC-SHA-256 rows now hoist key setup out of the timed loop and clone the keyed state per iteration, matching the treatment given to rscrypto, ring, and AWS-LC. The archival equivalence caveat no longer applies, so the aggregate is published as an equivalent-work claim. The chart clamped bars to a hardcoded 1.60x ceiling, which saturated AEAD at 1.61x and RSA at 1.65x to identical full-width bars and collapsed the sub-parity ECDSA row to zero width, rendering it indistinguishable from missing data. Bars now grow from a labeled 1.00x parity marker with losses extending left, and the axis derives its bounds from the data. --- README.md | 28 ++- assets/readme/perf.svg | 2 +- benchmark_results/OVERVIEW.md | 460 ++++++++++++++++++---------------- docs/benchmarking.md | 17 +- docs/platforms.md | 12 +- scripts/render_perf_chart.rs | 41 ++- 6 files changed, 308 insertions(+), 252 deletions(-) diff --git a/README.md b/README.md index 730050ca..26b1cb72 100644 --- a/README.md +++ b/README.md @@ -156,24 +156,28 @@ code from a binary. ## Performance -The published 2026-07-04 benchmark snapshot is historical. Its aggregate -geomeans are not equivalent-work performance claims because the historical -RustCrypto HMAC-SHA-256 rows included key setup while the compared rscrypto, -`ring`, and AWS-LC rows reused keyed state. The benchmark source now aligns -that setup, but a new aggregate requires a complete regenerated artifact. - -
-Historical 2026-07-04 scorecard (not an equivalent-work aggregate) +The 2026-08-18 snapshot covers eight Linux CI runners at commit `7eb44e9`. The +RustCrypto HMAC-SHA-256 key setup is now hoisted out of the timed loop, matching +the reusable keyed state given to rscrypto, `ring`, and AWS-LC, so these +aggregates are equivalent-work claims. Ratios are `external / rscrypto`; higher +is better. +

- rscrypto benchmark chart: 1.59x Linux and 1.37x Apple Silicon fastest-matched geomeans, checksums at 5.18x against crc-fast, crc, crc32fast, crc32c, and crc64fast, plus primitive geomean bars and M1 MBP Apple Silicon notes.

-
+ +3,780 of 6,144 fastest-external comparisons are wins and 5,475 are wins or ties, +for a 1.62x Linux geomean. Known losses: ECDSA P-256/P-384 regressed sharply on +IBM z16/s390x in this run and drags every ECDSA aggregate below parity (0.87x +across 128 rows; 1.19x-1.53x excluding that one runner), and +`rapidhash-stream/one-write` trails the `rapidhash` crate at 0.87x, mostly on +x86_64. Use individual shape-compatible rows for investigation and benchmark the -deployment workload on its target hardware. The correction, raw results, -methodology, and known losses are in +deployment workload on its target hardware. Raw results, methodology, and the +full loss list are in [`benchmark_results/OVERVIEW.md`](benchmark_results/OVERVIEW.md) and [`docs/benchmarking.md`](docs/benchmarking.md). diff --git a/assets/readme/perf.svg b/assets/readme/perf.svg index 1030bba0..40a68c56 100644 --- a/assets/readme/perf.svg +++ b/assets/readme/perf.svg @@ -2,4 +2,4 @@ text.sans { font-family: ui-sans-serif, -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; } text.mono { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, 'Liberation Mono', monospace; } text.hero { letter-spacing: -2px; paint-order: stroke fill; stroke: #0090FF; stroke-width: 0.7px; } -rscryptoGeomean speedups vs fastest matched competitors. Higher is better.Linux1.59x4,052 wins | 6,101 wins/ties | 6,750 casesApple Silicon1.37x382 wins | 708 wins/ties | 774 casesChecksums5.18x- Competitor Crates/Libs: crc-fast, crc, crc32fast, crc32c, crc64fastPrimitive GeomeansAEAD1.56xRSA1.55xECDSA1.45xHashes/MAC/XOF1.35xAuth/KDF1.25xPassword Hashing1.07xML-KEM1.49x- Linux Runners: AMD Zen 4/5; Intel Sapphire Rapids/Ice Lake; AWS Graviton 3/4; IBM POWER 10 and IBM Z16 (s390x); Rise RISC-V- macOS: MBP M1 10-Core, 16GB RAM - Local Dev Box- Fastest External: aws-lc-rs, ring, RustCrypto, BLAKE3, libcrux, crc-fast, etc. \ No newline at end of file +rscryptoGeomean speedups vs fastest matched competitors. Higher is better.Linux1.62x3,780 wins | 5,475 wins/ties | 6,144 casesApple Silicon1.37x382 wins | 708 wins/ties | 774 casesChecksums6.18x- Competitor Crates/Libs: crc-fast, crc, crc32fast, crc32c, crc64fastPrimitive GeomeansAEAD1.61xRSA1.65xECDSA0.87xHashes/MAC/XOF1.35xAuth/KDF1.28xPassword Hashing1.07xML-KEM1.55x1.00x- Linux Runners: AMD Zen 4/5; Intel Sapphire Rapids/Ice Lake; AWS Graviton 3/4; IBM POWER 10 and IBM Z16 (s390x)- macOS: MBP M1 10-Core, 16GB RAM - Local Dev Box- Fastest External: aws-lc-rs, ring, RustCrypto, BLAKE3, libcrux, crc-fast, etc. \ No newline at end of file diff --git a/benchmark_results/OVERVIEW.md b/benchmark_results/OVERVIEW.md index a7fad017..f55021e3 100644 --- a/benchmark_results/OVERVIEW.md +++ b/benchmark_results/OVERVIEW.md @@ -2,25 +2,35 @@ Sources: -- Linux benchmark CI run [#28710784737](https://github.com/loadingalias/rscrypto/actions/runs/28710784737), created 2026-07-04 15:26:12 UTC. -- Linux commit: `596498f0e07e869eac71fd31c157aa1b22186239`. -- Linux artifacts: nine successful `benchmark-*` artifacts extracted into `benchmark_results/2026-07-04/linux/*/results.txt`. -- Local macOS run: `benchmark_results/2026-07-04/macos/aarch64/results.txt` at commit `596498f0e07e869eac71fd31c157aa1b22186239`. +- Linux benchmark CI run [#32185659553](https://github.com/loadingalias/rscrypto/actions/runs/32185659553), created 2026-08-18 21:03:07 UTC. +- Linux commit: `7eb44e9a38ef7a031d9181dc8c4c0fad38f46504`. +- Linux artifacts: eight successful `benchmark-*` artifacts extracted into `benchmark_results/2026-08-18/linux/*/results.txt`. +- Local macOS run: `benchmark_results/2026-07-04/macos/aarch64/results.txt` at commit `596498f0e07e869eac71fd31c157aa1b22186239`, carried forward unchanged. - Local Ed25519 direct-secret before/after diagnostic, recorded below. -Scope: the 2026-07-04 nine-runner Linux CI benchmark matrix for commit `596498f`. Ratios are `external_crate_time / rscrypto_time`; higher is better. Wins are `>1.05x`, ties are `0.95x..1.05x`, and losses are `<0.95x`. Fastest-external comparisons keep only the fastest external implementation for each platform, primitive, operation, and input shape. Internal kernel, scratch-buffer, padding-only, cold-path, PHC roundtrip, parallel-scaling, threshold-selection, public-overhead, and phase-attribution microbenches are parsed as raw rows but excluded from external win/loss claims. The macOS local run is listed separately and is not mixed into Linux CI claims. +Scope: the 2026-08-18 eight-runner Linux CI benchmark matrix for commit `7eb44e9`. Ratios are `external_crate_time / rscrypto_time`; higher is better. Wins are `>1.05x`, ties are `0.95x..1.05x`, and losses are `<0.95x`. Fastest-external comparisons keep only the fastest external implementation for each platform, primitive, operation, and input shape. Internal kernel, scratch-buffer, padding-only, cold-path, PHC roundtrip, parallel-scaling, threshold-selection, public-overhead, and phase-attribution microbenches are parsed as raw rows but excluded from external win/loss claims. The macOS local run is listed separately and is not mixed into Linux CI claims. -This is a historical snapshot of commit `596498f`, not an inventory of the +This is a historical snapshot of commit `7eb44e9`, not an inventory of the current public API. Primitive rows remain as measured even when a later commit changes or removes that surface. -Equivalence correction: the historical RustCrypto HMAC-SHA-256 rows include -key setup inside the timed loop, while rscrypto, ring, and AWS-LC use reusable -keyed state. Consequently, the HMAC-SHA-256 row and aggregate statistics that -include it are archival measurements, not equivalent-work performance claims. -The current benchmark source precomputes the RustCrypto state; publish new -HMAC-SHA-256 and aggregate ratios only after a complete benchmark artifact is -regenerated. +Runner coverage change: this run has eight Linux runners. The RISE RISC-V +runner did not execute in run #32185659553, so every aggregate below is over +eight platforms rather than the nine in the 2026-07-04 snapshot. Row counts are +therefore not directly comparable to that snapshot; ratios and geomeans are. + +Equivalence correction resolved: the historical RustCrypto HMAC-SHA-256 rows +included key setup inside the timed loop. The current benchmark source hoists +`RustCryptoHmacSha256::new_from_slice` out of the timed loop and clones the +keyed state per iteration, matching the reusable-keyed-state treatment given to +rscrypto, ring, and AWS-LC. This artifact is a complete regenerated benchmark +pass, so the HMAC-SHA-256 rows and the aggregates that include them are +equivalent-work performance claims. + +Surface change since 2026-07-04: the rapidhash benchmark surface was collapsed. +The former `rapidhash-64`, `rapidhash-128`, and `rapidhash-v3-128` primitives no +longer exist; `rapidhash-v3-64`, `rapidhash-stream`, `rapidhash-buildhasher`, +`rapidhash-hash-one`, and `rapidhash-hashmap` are the current rows. Coverage note: this is a full Linux CI public benchmark pass. It includes checksum, hash, XOF, MAC, KDF, password-hashing, BLAKE2/BLAKE3, RSA import/verification, ECDSA P-256/P-384 signing and verification, Ed25519, X25519, AEAD, and ML-KEM-512/768/1024 keygen, encapsulation, and decapsulation rows. ML-KEM phase/arithmetic microbenches are present in the raw artifacts and intentionally excluded from release-level competitor claims. @@ -60,117 +70,121 @@ curation: | Scope | Pairs | W/T/L | Win % | Geomean | Median | | --- | --- | --- | --- | --- | --- | -| Linux CI: all matched performance pairs | 10,781 | 7,542/2,470/769 | 70% | 1.76x | 1.22x | -| Linux CI: fastest external per case | 6,750 | 4,052/2,049/649 | 60% | 1.59x | 1.11x | +| Linux CI: all matched performance pairs | 9,674 | 6,831/2,085/758 | 71% | 1.78x | 1.24x | +| Linux CI: fastest external per case | 6,144 | 3,780/1,695/669 | 62% | 1.62x | 1.12x | Snapshot summary: -- **Headline:** 4,052 of 6,750 matched Linux CI fastest-external comparisons are wins; 6,101 are wins or ties. Linux CI fastest-external geomean is 1.59x. -- **Checksums:** 5.18x geomean across 693 fastest-external rows; W/T/L is 517/115/61. -- **Hashes/MACs/XOFs:** 1.35x geomean across 3,726 fastest-external rows; W/T/L is 2,028/1,457/241. -- **Auth/KDF:** 1.25x geomean across 180 fastest-external rows; W/T/L is 159/20/1. -- **Password hashing:** 1.07x geomean across 135 fastest-external rows; W/T/L is 69/28/38. -- **Public-key:** 1.33x geomean across 333 fastest-external rows; W/T/L is 217/73/43. -- **RSA:** 1.55x geomean across 99 fastest-external rows; W/T/L is 89/2/8. -- **AEAD:** 1.56x geomean across 1,584 fastest-external rows; W/T/L is 973/354/257. -- **ML-KEM:** 1.49x geomean across 81 fastest-external rows; W/T/L is 68/4/9. -- **ECDSA P-256/P-384:** Linux CI 1.45x geomean across 144 fastest-external rows; W/T/L is 116/7/21. -- **Top current loss areas:** `argon2id-owasp` / `hash`: 0.97x geomean across 9 rows; W/T/L 4/1/4; pressure `rustcrypto` 4; `ed25519` / `verify`: 1.00x geomean across 36 rows; W/T/L 7/20/9; pressure `ring` 6, `dalek` 3; `mlkem1024` / `keygen`: 1.00x geomean across 9 rows; W/T/L 6/0/3; pressure `aws-lc-rs` 3; `blake2b256` / `streaming`: 1.02x geomean across 27 rows; W/T/L 10/16/1; pressure `rustcrypto` 1; `argon2id-small` / `hash`: 1.02x geomean across 27 rows; W/T/L 12/4/11; pressure `rustcrypto` 11. +- **Headline:** 3,780 of 6,144 matched Linux CI fastest-external comparisons are wins; 5,475 are wins or ties. Linux CI fastest-external geomean is 1.62x. +- **Checksums:** 6.18x geomean across 616 fastest-external rows; W/T/L is 476/118/22. +- **Hashes/MACs/XOFs:** 1.35x geomean across 3,456 fastest-external rows; W/T/L is 1,926/1,181/349. +- **Auth/KDF:** 1.28x geomean across 160 fastest-external rows; W/T/L is 140/20/0. +- **Password hashing:** 1.07x geomean across 120 fastest-external rows; W/T/L is 55/27/38. +- **Public-key:** 1.09x geomean across 296 fastest-external rows; W/T/L is 187/59/50. +- **RSA:** 1.65x geomean across 88 fastest-external rows; W/T/L is 86/2/0. +- **AEAD:** 1.61x geomean across 1,408 fastest-external rows; W/T/L is 910/288/210. +- **ML-KEM:** 1.55x geomean across 72 fastest-external rows; W/T/L is 64/0/8. +- **ECDSA P-256/P-384:** Linux CI 0.87x geomean across 128 fastest-external rows; W/T/L is 88/7/33. +- **Top current loss areas:** `ecdsa-p384` / `sign`: 0.70x geomean across 32 rows; W/T/L is 12/0/20; pressure `aws-lc-rs` 16, `rustcrypto-p384` 4; `ecdsa-p256` / `verify`: 0.84x geomean across 32 rows; W/T/L is 20/7/5; pressure `rustcrypto-p256` 4, `aws-lc-rs` 1; `rapidhash-stream` / `one-write`: 0.87x geomean across 88 rows; W/T/L is 27/25/36; pressure `rapidhash` 36; `ecdsa-p256` / `sign`: 0.91x geomean across 32 rows; W/T/L is 28/0/4; pressure `ring` 4; `argon2id-owasp` / `hash`: 0.98x geomean across 8 rows; W/T/L is 3/1/4; pressure `rustcrypto` 3, `dryoc` 1. ## Coverage Matrix | Platform | Raw Criterion rows | All pairs | Fastest rows | W/T/L | Win % | Geomean | Median | | --- | --- | --- | --- | --- | --- | --- | --- | -| AMD Zen4 | 2,356 | 1,251 | 750 | 513/189/48 | 68% | 1.50x | 1.15x | -| AMD Zen5 | 2,356 | 1,251 | 750 | 434/249/67 | 58% | 1.49x | 1.09x | -| AWS Graviton3 | 2,367 | 1,251 | 750 | 360/298/92 | 48% | 1.37x | 1.04x | -| AWS Graviton4 | 2,367 | 1,251 | 750 | 362/332/56 | 48% | 1.37x | 1.05x | -| IBM Power10 | 2,107 | 1,012 | 750 | 389/324/37 | 52% | 1.89x | 1.06x | -| IBM z16/s390x | 2,107 | 1,012 | 750 | 603/87/60 | 80% | 3.15x | 2.34x | -| Intel Ice Lake | 2,356 | 1,251 | 750 | 523/154/73 | 70% | 1.49x | 1.19x | -| Intel Sapphire Rapids | 2,356 | 1,251 | 750 | 533/149/68 | 71% | 1.63x | 1.20x | -| RISE RISC-V | 2,356 | 1,251 | 750 | 335/267/148 | 45% | 1.09x | 1.03x | +| AMD Zen4 | 2,304 | 1,269 | 768 | 525/171/72 | 68% | 1.47x | 1.14x | +| AMD Zen5 | 2,304 | 1,269 | 768 | 447/245/76 | 58% | 1.47x | 1.10x | +| AWS Graviton3 | 2,308 | 1,269 | 768 | 367/287/114 | 48% | 1.36x | 1.04x | +| AWS Graviton4 | 2,308 | 1,269 | 768 | 366/337/65 | 48% | 1.37x | 1.04x | +| IBM Power10 | 2,055 | 1,030 | 768 | 400/302/66 | 52% | 1.83x | 1.06x | +| IBM z16/s390x | 2,055 | 1,030 | 768 | 620/67/81 | 81% | 2.77x | 2.19x | +| Intel Ice Lake | 2,304 | 1,269 | 768 | 517/137/114 | 67% | 1.45x | 1.17x | +| Intel Sapphire Rapids | 2,304 | 1,269 | 768 | 538/149/81 | 70% | 1.60x | 1.18x | ## Category Summary | Category | Rows | W/T/L | Win % | Geomean | Median | | --- | --- | --- | --- | --- | --- | -| Checksums | 693 | 517/115/61 | 75% | 5.18x | 2.44x | -| Hashes/MACs/XOFs | 3,726 | 2,028/1,457/241 | 54% | 1.35x | 1.07x | -| Auth/KDF | 180 | 159/20/1 | 88% | 1.25x | 1.13x | -| Password hashing | 135 | 69/28/38 | 51% | 1.07x | 1.05x | -| Public-key | 333 | 217/73/43 | 65% | 1.33x | 1.17x | -| RSA | 99 | 89/2/8 | 90% | 1.55x | 1.18x | -| AEAD | 1,584 | 973/354/257 | 61% | 1.56x | 1.15x | +| Checksums | 616 | 476/118/22 | 77% | 6.18x | 3.17x | +| Hashes/MACs/XOFs | 3,456 | 1,926/1,181/349 | 56% | 1.35x | 1.08x | +| Auth/KDF | 160 | 140/20/0 | 88% | 1.28x | 1.13x | +| Password hashing | 120 | 55/27/38 | 46% | 1.07x | 1.02x | +| Public-key | 296 | 187/59/50 | 63% | 1.09x | 1.14x | +| RSA | 88 | 86/2/0 | 98% | 1.65x | 1.20x | +| AEAD | 1,408 | 910/288/210 | 65% | 1.61x | 1.21x | ## BLAKE3 Summary -BLAKE3 rows come from Linux CI run [#28710784737](https://github.com/loadingalias/rscrypto/actions/runs/28710784737). All-pair and fastest-external BLAKE3 metrics are identical because official `blake3` is the only external implementation in this bench. +BLAKE3 rows come from Linux CI run [#32185659553](https://github.com/loadingalias/rscrypto/actions/runs/32185659553). All-pair and fastest-external BLAKE3 metrics are identical because official `blake3` is the only external implementation in this bench. | Scope | Rows | W/T/L | Geomean | Median | | --- | --- | --- | --- | --- | -| All Linux BLAKE3 rows | 432 | 234/168/30 | 1.41x | 1.08x | -| x86_64 | 192 | 92/90/10 | 1.26x | 1.05x | -| AArch64 | 96 | 44/46/6 | 1.44x | 1.05x | +| All Linux BLAKE3 rows | 384 | 187/134/63 | 1.35x | 1.04x | +| x86_64 | 192 | 79/89/24 | 1.18x | 1.02x | +| AArch64 | 96 | 44/36/16 | 1.40x | 1.04x | | Platform | Rows | W/T/L | Geomean | Median | | --- | --- | --- | --- | --- | -| AMD Zen4 | 48 | 23/23/2 | 1.34x | 1.05x | -| AMD Zen5 | 48 | 21/25/2 | 1.32x | 1.03x | -| AWS Graviton3 | 48 | 22/21/5 | 1.40x | 0.99x | -| AWS Graviton4 | 48 | 22/25/1 | 1.47x | 1.05x | -| IBM Power10 | 48 | 39/9/0 | 1.98x | 1.85x | -| IBM z16/s390x | 48 | 36/5/7 | 1.81x | 2.10x | -| Intel Ice Lake | 48 | 19/26/3 | 1.17x | 1.02x | -| Intel Sapphire Rapids | 48 | 29/16/3 | 1.24x | 1.08x | -| RISE RISC-V | 48 | 23/18/7 | 1.15x | 1.02x | +| AMD Zen4 | 48 | 20/22/6 | 1.24x | 1.01x | +| AMD Zen5 | 48 | 18/27/3 | 1.27x | 1.02x | +| AWS Graviton3 | 48 | 22/15/11 | 1.36x | 0.98x | +| AWS Graviton4 | 48 | 22/21/5 | 1.44x | 1.04x | +| IBM Power10 | 48 | 32/6/10 | 1.76x | 1.12x | +| IBM z16/s390x | 48 | 32/3/13 | 1.69x | 1.69x | +| Intel Ice Lake | 48 | 19/21/8 | 1.09x | 1.00x | +| Intel Sapphire Rapids | 48 | 22/19/7 | 1.13x | 1.03x | | Operation | Rows | W/T/L | Geomean | Median | | --- | --- | --- | --- | --- | -| `oneshot` | 99 | 45/47/7 | 1.32x | 1.02x | -| `keyed` | 99 | 46/45/8 | 1.34x | 1.03x | -| `derive-key` | 99 | 77/20/2 | 1.76x | 1.86x | -| `streaming` | 36 | 11/21/4 | 1.16x | 1.01x | -| `xof` | 99 | 55/35/9 | 1.35x | 1.07x | +| `oneshot` | 88 | 35/35/18 | 1.33x | 1.00x | +| `keyed` | 88 | 27/21/40 | 1.20x | 0.95x | +| `derive-key` | 88 | 65/21/2 | 1.59x | 1.53x | +| `streaming` | 32 | 10/21/1 | 1.21x | 1.02x | +| `xof` | 88 | 50/36/2 | 1.37x | 1.07x | ## ML-KEM Summary -ML-KEM public coverage is complete for the CI-selected primitive set: ML-KEM-512, ML-KEM-768, and ML-KEM-1024 each include keygen, encapsulate, and decapsulate on all nine Linux platforms. POWER10 and s390x do not have `aws-lc-rs` ML-KEM rows in this artifact set, but still have rscrypto plus `libcrux`, `fips203`, and RustCrypto comparison rows for every public operation. +ML-KEM public coverage is complete for the CI-selected primitive set: ML-KEM-512, ML-KEM-768, and ML-KEM-1024 each include keygen, encapsulate, and decapsulate on all eight Linux platforms. POWER10 and s390x do not have `aws-lc-rs` ML-KEM rows in this artifact set, but still have rscrypto plus `libcrux`, `fips203`, and RustCrypto comparison rows for every public operation. | Platform | Raw ML-KEM rows | Fastest rows | W/T/L | Geomean | Median | Fastest external split | | --- | --- | --- | --- | --- | --- | --- | -| AMD Zen4 | 45 | 9 | 9/0/0 | 1.81x | 1.70x | `libcrux` 7, `aws-lc-rs` 2 | -| AMD Zen5 | 45 | 9 | 9/0/0 | 1.93x | 1.75x | `libcrux` 9 | -| AWS Graviton3 | 45 | 9 | 5/0/4 | 1.08x | 1.13x | `aws-lc-rs` 9 | -| AWS Graviton4 | 45 | 9 | 5/0/4 | 1.08x | 1.14x | `aws-lc-rs` 9 | -| IBM Power10 | 36 | 9 | 9/0/0 | 1.47x | 1.58x | `libcrux` 9 | -| IBM z16/s390x | 36 | 9 | 9/0/0 | 1.58x | 1.64x | `libcrux` 9 | -| Intel Ice Lake | 45 | 9 | 9/0/0 | 1.79x | 1.74x | `libcrux` 7, `aws-lc-rs` 2 | -| Intel Sapphire Rapids | 45 | 9 | 9/0/0 | 1.85x | 1.82x | `aws-lc-rs` 5, `libcrux` 4 | -| RISE RISC-V | 45 | 9 | 4/4/1 | 1.13x | 1.03x | `aws-lc-rs` 9 | +| AMD Zen4 | 45 | 9 | 9/0/0 | 1.83x | 1.82x | `libcrux` 7, `aws-lc-rs` 2 | +| AMD Zen5 | 45 | 9 | 9/0/0 | 1.95x | 1.91x | `libcrux` 9 | +| AWS Graviton3 | 45 | 9 | 5/0/4 | 1.09x | 1.12x | `aws-lc-rs` 9 | +| AWS Graviton4 | 45 | 9 | 5/0/4 | 1.08x | 1.18x | `aws-lc-rs` 9 | +| IBM Power10 | 36 | 9 | 9/0/0 | 1.41x | 1.53x | `libcrux` 9 | +| IBM z16/s390x | 36 | 9 | 9/0/0 | 1.68x | 1.74x | `libcrux` 9 | +| Intel Ice Lake | 45 | 9 | 9/0/0 | 1.80x | 1.75x | `libcrux` 7, `aws-lc-rs` 2 | +| Intel Sapphire Rapids | 45 | 9 | 9/0/0 | 1.84x | 1.80x | `aws-lc-rs` 7, `libcrux` 2 | | Primitive/op | Rows | W/T/L | Win % | Geomean | Median | Pressure | | --- | --- | --- | --- | --- | --- | --- | -| `mlkem1024` / `decapsulate` | 9 | 9/0/0 | 100% | 1.59x | 1.67x | none | -| `mlkem1024` / `encapsulate` | 9 | 9/0/0 | 100% | 2.39x | 1.98x | none | -| `mlkem1024` / `keygen` | 9 | 6/0/3 | 67% | 1.00x | 1.09x | `aws-lc-rs` 3 | -| `mlkem512` / `decapsulate` | 9 | 6/1/2 | 67% | 1.33x | 1.51x | `aws-lc-rs` 2 | -| `mlkem512` / `encapsulate` | 9 | 9/0/0 | 100% | 1.86x | 1.70x | none | -| `mlkem512` / `keygen` | 9 | 6/1/2 | 67% | 1.08x | 1.18x | `aws-lc-rs` 2 | -| `mlkem768` / `decapsulate` | 9 | 8/1/0 | 89% | 1.49x | 1.70x | none | -| `mlkem768` / `encapsulate` | 9 | 9/0/0 | 100% | 2.27x | 2.02x | none | -| `mlkem768` / `keygen` | 9 | 6/1/2 | 67% | 1.04x | 1.12x | `aws-lc-rs` 2 | +| `mlkem1024` / `decapsulate` | 8 | 8/0/0 | 100% | 1.70x | 1.86x | none | +| `mlkem1024` / `encapsulate` | 8 | 8/0/0 | 100% | 2.51x | 2.63x | none | +| `mlkem1024` / `keygen` | 8 | 6/0/2 | 75% | 1.02x | 1.13x | `aws-lc-rs` 2 | +| `mlkem512` / `decapsulate` | 8 | 6/0/2 | 75% | 1.41x | 1.59x | `aws-lc-rs` 2 | +| `mlkem512` / `encapsulate` | 8 | 8/0/0 | 100% | 1.94x | 2.17x | none | +| `mlkem512` / `keygen` | 8 | 6/0/2 | 75% | 1.09x | 1.22x | `aws-lc-rs` 2 | +| `mlkem768` / `decapsulate` | 8 | 8/0/0 | 100% | 1.58x | 1.75x | none | +| `mlkem768` / `encapsulate` | 8 | 8/0/0 | 100% | 2.33x | 2.54x | none | +| `mlkem768` / `keygen` | 8 | 6/0/2 | 75% | 1.06x | 1.13x | `aws-lc-rs` 2 | ## ECDSA Summary ECDSA signing includes both deterministic and blinded rscrypto rows in raw results; aggregate fastest-external comparisons use the fastest rscrypto row for the exact case. Constant-time release evidence is tracked separately by `ct.toml` and CT workflow artifacts. +Regression: every ECDSA aggregate in this snapshot is dominated by a single +platform. On IBM z16/s390x, P-256 signing went from 137.10 µs (2026-07-04) to +8,889.30 µs, and P-384 signing from 562.91 µs to 34,557.00 µs, while the +external crates on the same runner moved by less than 1.4x. Excluding s390x, the +seven-runner geomeans are `ecdsa-p256` / `sign` 1.33x, `ecdsa-p256` / `verify` +1.19x, `ecdsa-p384` / `sign` 1.01x, and `ecdsa-p384` / `verify` 1.53x. + | Operation | Rows | W/T/L | Geomean | Median | | --- | --- | --- | --- | --- | -| `ecdsa-p256` / `sign` | 36 | 36/0/0 | 1.45x | 1.35x | -| `ecdsa-p256` / `verify` | 36 | 28/7/1 | 1.59x | 1.15x | -| `ecdsa-p384` / `sign` | 36 | 16/0/20 | 1.07x | 0.90x | -| `ecdsa-p384` / `verify` | 36 | 36/0/0 | 1.81x | 1.39x | +| `ecdsa-p256` / `sign` | 32 | 28/0/4 | 0.91x | 1.30x | +| `ecdsa-p256` / `verify` | 32 | 20/7/5 | 0.84x | 1.08x | +| `ecdsa-p384` / `sign` | 32 | 12/0/20 | 0.70x | 0.83x | +| `ecdsa-p384` / `verify` | 32 | 28/0/4 | 1.08x | 1.36x | ## Primitive Summary @@ -178,146 +192,149 @@ Linux CI primitives with matched exact `rscrypto` comparisons. Fastest columns a | Primitive | Fastest rows | Fastest W/T/L | Fastest geomean | All pairs | All W/T/L | All geomean | | --- | --- | --- | --- | --- | --- | --- | -| `argon2id-owasp` | 9 | 4/1/4 | 0.97x | 18 | 11/2/5 | 1.27x | -| `argon2id-small` | 27 | 12/4/11 | 1.02x | 45 | 29/5/11 | 1.35x | -| `argon2d-small` | 27 | 13/6/8 | 1.03x | 27 | 13/6/8 | 1.03x | -| `x25519` | 18 | 2/16/0 | 1.03x | 50 | 33/17/0 | 1.54x | -| `argon2i-small` | 27 | 13/2/12 | 1.03x | 45 | 30/2/13 | 1.35x | -| `rapidhash-v3-64` | 99 | 27/53/19 | 1.05x | 99 | 27/53/19 | 1.05x | -| `blake2b256` | 225 | 120/100/5 | 1.07x | 351 | 238/108/5 | 1.31x | -| `rapidhash-v3-128` | 99 | 34/44/21 | 1.07x | 99 | 34/44/21 | 1.07x | -| `blake2b512` | 198 | 109/82/7 | 1.07x | 297 | 207/83/7 | 1.31x | -| `xxh3-128` | 99 | 29/51/19 | 1.08x | 99 | 29/51/19 | 1.08x | -| `scrypt-owasp` | 9 | 5/2/2 | 1.09x | 9 | 5/2/2 | 1.09x | -| `xxh3-64` | 99 | 33/47/19 | 1.10x | 99 | 33/47/19 | 1.10x | -| `rapidhash-64` | 99 | 32/51/16 | 1.10x | 99 | 32/51/16 | 1.10x | -| `chacha20-poly1305` | 198 | 83/108/7 | 1.10x | 550 | 344/198/8 | 1.29x | -| `blake2s256` | 225 | 117/107/1 | 1.10x | 225 | 117/107/1 | 1.10x | -| `ed25519` | 90 | 31/46/13 | 1.11x | 290 | 204/69/17 | 1.36x | -| `blake2s128` | 198 | 118/79/1 | 1.11x | 198 | 118/79/1 | 1.11x | -| `rapidhash-128` | 99 | 38/56/5 | 1.14x | 99 | 38/56/5 | 1.14x | -| `rsa-8192` | 18 | 14/2/2 | 1.19x | 32 | 28/2/2 | 1.25x | -| `scrypt-small` | 36 | 22/13/1 | 1.21x | 36 | 22/13/1 | 1.21x | -| `hmac-sha512` | 99 | 37/54/8 | 1.23x | 275 | 165/97/13 | 1.29x | -| `hmac-sha384` | 99 | 38/52/9 | 1.24x | 275 | 167/94/14 | 1.29x | -| `hkdf-sha384` | 36 | 33/2/1 | 1.24x | 100 | 96/3/1 | 1.53x | -| `sha256` | 117 | 41/59/17 | 1.24x | 293 | 164/99/30 | 1.54x | -| `sha512` | 117 | 45/61/11 | 1.25x | 293 | 172/104/17 | 1.27x | -| `pbkdf2-sha512` | 54 | 47/7/0 | 1.25x | 150 | 143/7/0 | 1.33x | -| `sha384` | 99 | 43/45/11 | 1.25x | 275 | 169/89/17 | 1.28x | -| `pbkdf2-sha256` | 54 | 50/4/0 | 1.25x | 150 | 146/4/0 | 1.65x | -| `hkdf-sha256` | 36 | 29/7/0 | 1.26x | 100 | 93/7/0 | 1.87x | -| `ascon-hash256` | 99 | 54/43/2 | 1.27x | 99 | 54/43/2 | 1.27x | -| `sha512-256` | 99 | 53/46/0 | 1.29x | 99 | 53/46/0 | 1.29x | -| `ascon-aead128` | 198 | 148/50/0 | 1.33x | 198 | 148/50/0 | 1.33x | -| `hmac-sha256` | 117 | 64/39/14 | 1.34x | 293 | 193/76/24 | 1.68x | -| `aegis-256` | 198 | 89/88/21 | 1.37x | 198 | 89/88/21 | 1.37x | -| `ascon-xof128` | 99 | 75/24/0 | 1.37x | 99 | 75/24/0 | 1.37x | -| `mlkem512` | 27 | 21/2/4 | 1.39x | 102 | 96/2/4 | 2.67x | -| `xchacha20-poly1305` | 198 | 182/16/0 | 1.39x | 198 | 182/16/0 | 1.39x | -| `ecdsa-p384` | 72 | 52/0/20 | 1.39x | 200 | 180/0/20 | 3.17x | -| `blake3` | 432 | 234/168/30 | 1.41x | 432 | 234/168/30 | 1.41x | -| `ecdsa-p256` | 72 | 64/7/1 | 1.52x | 200 | 188/11/1 | 2.40x | -| `mlkem768` | 27 | 23/2/2 | 1.52x | 102 | 98/2/2 | 3.11x | -| `mlkem1024` | 27 | 24/0/3 | 1.56x | 102 | 99/0/3 | 3.27x | -| `rsa-4096` | 27 | 25/0/2 | 1.56x | 59 | 57/0/2 | 2.52x | -| `crc32c` | 99 | 45/37/17 | 1.61x | 198 | 140/40/18 | 2.19x | -| `crc32` | 99 | 49/31/19 | 1.66x | 198 | 144/35/19 | 2.26x | -| `rsa-3072` | 27 | 25/0/2 | 1.66x | 59 | 57/0/2 | 2.59x | -| `aes-128-gcm` | 198 | 97/45/56 | 1.68x | 550 | 413/54/83 | 1.89x | -| `rsa-2048` | 27 | 25/0/2 | 1.71x | 59 | 57/0/2 | 2.63x | -| `aes-256-gcm` | 198 | 97/40/61 | 1.71x | 550 | 405/52/93 | 1.92x | -| `kmac256` | 99 | 65/18/16 | 1.75x | 99 | 65/18/16 | 1.75x | -| `cshake256` | 99 | 63/28/8 | 1.78x | 99 | 63/28/8 | 1.78x | -| `shake128` | 99 | 69/29/1 | 1.85x | 99 | 69/29/1 | 1.85x | -| `shake256` | 99 | 70/29/0 | 1.87x | 99 | 70/29/0 | 1.87x | -| `sha224` | 99 | 52/46/1 | 1.87x | 99 | 52/46/1 | 1.87x | -| `aes-128-gcm-siv` | 198 | 138/3/57 | 2.02x | 352 | 271/16/65 | 2.79x | -| `sha3-256` | 117 | 104/13/0 | 2.11x | 117 | 104/13/0 | 2.11x | -| `sha3-224` | 99 | 87/12/0 | 2.13x | 99 | 87/12/0 | 2.13x | -| `crc64-nvme` | 99 | 56/33/10 | 2.14x | 99 | 56/33/10 | 2.14x | -| `aes-256-gcm-siv` | 198 | 139/4/55 | 2.15x | 352 | 292/5/55 | 3.04x | -| `sha3-384` | 99 | 89/10/0 | 2.16x | 99 | 89/10/0 | 2.16x | -| `sha3-512` | 99 | 88/11/0 | 2.20x | 99 | 88/11/0 | 2.20x | -| `crc64-xz` | 99 | 76/12/11 | 2.39x | 99 | 76/12/11 | 2.39x | -| `crc24-openpgp` | 99 | 95/2/2 | 13.25x | 99 | 95/2/2 | 13.25x | -| `crc16-ccitt` | 99 | 98/0/1 | 23.12x | 99 | 98/0/1 | 23.12x | -| `crc16-ibm` | 99 | 98/0/1 | 24.01x | 99 | 98/0/1 | 24.01x | +| `ecdsa-p384` | 64 | 40/0/24 | 0.87x | 176 | 144/0/32 | 2.27x | +| `ecdsa-p256` | 64 | 48/7/9 | 0.87x | 176 | 148/11/17 | 1.57x | +| `rapidhash-stream` | 176 | 61/33/82 | 0.92x | 176 | 61/33/82 | 0.92x | +| `argon2id-owasp` | 8 | 3/1/4 | 0.98x | 16 | 7/4/5 | 1.25x | +| `xxh3-buildhasher` | 88 | 41/12/35 | 0.99x | 88 | 41/12/35 | 0.99x | +| `x25519` | 16 | 3/13/0 | 1.02x | 44 | 31/13/0 | 1.58x | +| `argon2i-small` | 24 | 10/3/11 | 1.03x | 40 | 26/3/11 | 1.34x | +| `argon2id-small` | 24 | 10/3/11 | 1.03x | 40 | 25/4/11 | 1.35x | +| `argon2d-small` | 24 | 10/5/9 | 1.04x | 24 | 10/5/9 | 1.04x | +| `rapidhash-v3-64` | 88 | 21/45/22 | 1.05x | 88 | 21/45/22 | 1.05x | +| `blake2b256` | 200 | 101/99/0 | 1.07x | 312 | 204/108/0 | 1.31x | +| `scrypt-owasp` | 8 | 4/2/2 | 1.08x | 8 | 4/2/2 | 1.08x | +| `blake2b512` | 176 | 106/69/1 | 1.08x | 264 | 194/69/1 | 1.33x | +| `blake2s256` | 200 | 114/86/0 | 1.11x | 200 | 114/86/0 | 1.11x | +| `chacha20-poly1305` | 176 | 75/101/0 | 1.12x | 484 | 304/180/0 | 1.32x | +| `xxh3-128` | 88 | 34/42/12 | 1.13x | 88 | 34/42/12 | 1.13x | +| `xxh3-64` | 88 | 34/34/20 | 1.13x | 88 | 34/34/20 | 1.13x | +| `blake2s128` | 176 | 113/63/0 | 1.13x | 176 | 113/63/0 | 1.13x | +| `ed25519` | 80 | 32/39/9 | 1.14x | 256 | 194/48/14 | 1.41x | +| `xxh3-hashmap` | 8 | 7/1/0 | 1.15x | 8 | 7/1/0 | 1.15x | +| `scrypt-small` | 32 | 18/13/1 | 1.18x | 32 | 18/13/1 | 1.18x | +| `rapidhash-buildhasher` | 88 | 44/29/15 | 1.19x | 88 | 44/29/15 | 1.19x | +| `aegis-256` | 176 | 81/65/30 | 1.23x | 176 | 81/65/30 | 1.23x | +| `hmac-sha256` | 104 | 42/36/26 | 1.24x | 258 | 144/78/36 | 1.60x | +| `hmac-sha384` | 88 | 28/49/11 | 1.24x | 242 | 133/93/16 | 1.29x | +| `hmac-sha512` | 88 | 32/44/12 | 1.27x | 242 | 137/88/17 | 1.31x | +| `sha256` | 104 | 44/46/14 | 1.27x | 258 | 143/89/26 | 1.60x | +| `hkdf-sha384` | 32 | 29/3/0 | 1.27x | 88 | 85/3/0 | 1.59x | +| `rsa-8192` | 16 | 14/2/0 | 1.28x | 28 | 26/2/0 | 1.33x | +| `hkdf-sha256` | 32 | 27/5/0 | 1.28x | 88 | 83/5/0 | 1.93x | +| `pbkdf2-sha256` | 48 | 43/5/0 | 1.28x | 132 | 127/5/0 | 1.71x | +| `pbkdf2-sha512` | 48 | 41/7/0 | 1.28x | 132 | 125/7/0 | 1.34x | +| `sha512` | 104 | 48/51/5 | 1.29x | 258 | 160/88/10 | 1.31x | +| `sha384` | 88 | 43/39/6 | 1.30x | 242 | 151/80/11 | 1.32x | +| `ascon-hash256` | 88 | 56/31/1 | 1.30x | 88 | 56/31/1 | 1.30x | +| `sha512-256` | 88 | 50/38/0 | 1.33x | 88 | 50/38/0 | 1.33x | +| `blake3` | 384 | 187/134/63 | 1.35x | 384 | 187/134/63 | 1.35x | +| `ascon-aead128` | 176 | 136/39/1 | 1.39x | 176 | 136/39/1 | 1.39x | +| `ascon-xof128` | 88 | 66/20/2 | 1.39x | 88 | 66/20/2 | 1.39x | +| `xchacha20-poly1305` | 176 | 173/3/0 | 1.43x | 176 | 173/3/0 | 1.43x | +| `mlkem512` | 24 | 20/0/4 | 1.44x | 90 | 86/0/4 | 2.90x | +| `rapidhash-hash-one` | 24 | 18/4/2 | 1.47x | 24 | 18/4/2 | 1.47x | +| `mlkem768` | 24 | 22/0/2 | 1.57x | 90 | 88/0/2 | 3.38x | +| `rapidhash-hashmap` | 24 | 24/0/0 | 1.61x | 24 | 24/0/0 | 1.61x | +| `mlkem1024` | 24 | 22/0/2 | 1.63x | 90 | 88/0/2 | 3.60x | +| `rsa-4096` | 24 | 24/0/0 | 1.70x | 52 | 52/0/0 | 2.69x | +| `crc32c` | 88 | 42/38/8 | 1.73x | 176 | 130/38/8 | 2.41x | +| `rsa-3072` | 24 | 24/0/0 | 1.75x | 52 | 52/0/0 | 2.73x | +| `rsa-2048` | 24 | 24/0/0 | 1.79x | 52 | 52/0/0 | 2.77x | +| `aes-128-gcm` | 176 | 96/42/38 | 1.80x | 484 | 390/50/44 | 2.01x | +| `crc32` | 88 | 47/33/8 | 1.80x | 176 | 133/35/8 | 2.51x | +| `aes-256-gcm` | 176 | 94/36/46 | 1.83x | 484 | 382/44/58 | 2.02x | +| `kmac256` | 88 | 58/19/11 | 1.86x | 88 | 58/19/11 | 1.86x | +| `cshake256` | 88 | 58/21/9 | 1.90x | 88 | 58/21/9 | 1.90x | +| `shake128` | 88 | 58/30/0 | 1.94x | 88 | 58/30/0 | 1.94x | +| `shake256` | 88 | 63/25/0 | 1.98x | 88 | 63/25/0 | 1.98x | +| `sha224` | 88 | 51/37/0 | 2.01x | 88 | 51/37/0 | 2.01x | +| `aes-128-gcm-siv` | 176 | 127/1/48 | 2.20x | 308 | 237/16/55 | 2.92x | +| `sha3-224` | 88 | 77/11/0 | 2.27x | 88 | 77/11/0 | 2.27x | +| `sha3-256` | 104 | 91/13/0 | 2.28x | 104 | 91/13/0 | 2.28x | +| `aes-256-gcm-siv` | 176 | 128/1/47 | 2.34x | 308 | 259/2/47 | 3.16x | +| `crc64-nvme` | 88 | 52/35/1 | 2.34x | 88 | 52/35/1 | 2.34x | +| `sha3-384` | 88 | 79/9/0 | 2.35x | 88 | 79/9/0 | 2.35x | +| `sha3-512` | 88 | 77/11/0 | 2.38x | 88 | 77/11/0 | 2.38x | +| `crc64-xz` | 88 | 73/12/3 | 2.78x | 88 | 73/12/3 | 2.78x | +| `crc24-openpgp` | 88 | 86/0/2 | 17.62x | 88 | 86/0/2 | 17.62x | +| `crc16-ccitt` | 88 | 88/0/0 | 30.24x | 88 | 88/0/0 | 30.24x | +| `crc16-ibm` | 88 | 88/0/0 | 32.07x | 88 | 88/0/0 | 32.07x | ## Linux Worst Individual Rows | Platform | Case | Fastest external | Ratio | | --- | --- | --- | --- | -| RISE RISC-V | `xxh3-64 / 0` | `xxhash-rust` | 0.43x | -| Intel Sapphire Rapids | `aes-256-gcm / encrypt / 32` | `rustcrypto` | 0.46x | -| Intel Ice Lake | `aes-256-gcm-siv / encrypt / 32` | `aws-lc-rs` | 0.48x | -| Intel Sapphire Rapids | `aes-128-gcm / encrypt / 32` | `rustcrypto` | 0.48x | -| Intel Ice Lake | `aes-256-gcm-siv / encrypt / 0` | `aws-lc-rs` | 0.48x | -| Intel Sapphire Rapids | `aes-256-gcm-siv / encrypt / 32` | `aws-lc-rs` | 0.49x | -| Intel Sapphire Rapids | `aes-128-gcm-siv / encrypt / 0` | `aws-lc-rs` | 0.49x | -| Intel Ice Lake | `aes-128-gcm-siv / encrypt / 32` | `aws-lc-rs` | 0.49x | -| Intel Sapphire Rapids | `aes-128-gcm-siv / encrypt / 32` | `aws-lc-rs` | 0.49x | -| AMD Zen4 | `aes-256-gcm / encrypt / 32` | `rustcrypto` | 0.49x | -| Intel Ice Lake | `aes-256-gcm / encrypt / 32` | `rustcrypto` | 0.49x | -| Intel Sapphire Rapids | `aes-256-gcm-siv / encrypt / 0` | `aws-lc-rs` | 0.49x | +| IBM z16/s390x | `ecdsa-p256 / sign / 1024` | `ring` | 0.05x | +| IBM z16/s390x | `ecdsa-p384 / sign / 16384` | `rustcrypto-p384` | 0.05x | +| IBM z16/s390x | `ecdsa-p384 / sign / 1024` | `rustcrypto-p384` | 0.05x | +| IBM z16/s390x | `ecdsa-p384 / sign / 0` | `rustcrypto-p384` | 0.06x | +| IBM z16/s390x | `ecdsa-p384 / sign / 32` | `rustcrypto-p384` | 0.06x | +| IBM z16/s390x | `ecdsa-p256 / sign / 0` | `ring` | 0.06x | +| IBM z16/s390x | `ecdsa-p256 / sign / 32` | `ring` | 0.06x | +| IBM z16/s390x | `ecdsa-p256 / verify / 32` | `rustcrypto-p256` | 0.06x | +| IBM z16/s390x | `ecdsa-p256 / verify / 1024` | `rustcrypto-p256` | 0.07x | +| IBM z16/s390x | `ecdsa-p256 / sign / 16384` | `ring` | 0.07x | +| IBM z16/s390x | `ecdsa-p256 / verify / 0` | `rustcrypto-p256` | 0.07x | +| IBM z16/s390x | `ecdsa-p384 / verify / 1024` | `rustcrypto-p384` | 0.09x | ## Linux Strongest Individual Rows | Platform | Case | Fastest external | Ratio | | --- | --- | --- | --- | -| Intel Sapphire Rapids | `crc16-ccitt / 262144` | `crc` | 214.89x | -| Intel Sapphire Rapids | `crc16-ibm / 262144` | `crc` | 209.87x | -| Intel Sapphire Rapids | `crc16-ibm / 16384` | `crc` | 208.50x | -| Intel Sapphire Rapids | `crc16-ccitt / 16384` | `crc` | 207.55x | -| Intel Sapphire Rapids | `crc16-ccitt / 4096` | `crc` | 184.38x | -| Intel Sapphire Rapids | `crc16-ccitt / 65536` | `crc` | 183.70x | -| Intel Sapphire Rapids | `crc16-ibm / 65536` | `crc` | 179.68x | -| Intel Sapphire Rapids | `crc16-ccitt / 1048576` | `crc` | 178.10x | -| IBM Power10 | `crc16-ibm / 262144` | `crc` | 176.65x | -| IBM Power10 | `crc16-ibm / 1048576` | `crc` | 176.64x | -| IBM Power10 | `crc16-ccitt / 262144` | `crc` | 176.33x | -| IBM Power10 | `crc16-ccitt / 1048576` | `crc` | 176.09x | +| Intel Sapphire Rapids | `crc16-ibm / 262144` | `crc` | 212.60x | +| Intel Sapphire Rapids | `crc16-ccitt / 262144` | `crc` | 209.27x | +| Intel Sapphire Rapids | `crc16-ccitt / 16384` | `crc` | 206.40x | +| Intel Sapphire Rapids | `crc16-ibm / 16384` | `crc` | 198.48x | +| Intel Sapphire Rapids | `crc16-ibm / 1048576` | `crc` | 187.52x | +| Intel Sapphire Rapids | `crc16-ibm / 4096` | `crc` | 178.55x | +| Intel Sapphire Rapids | `crc16-ibm / 65536` | `crc` | 178.28x | +| Intel Sapphire Rapids | `crc16-ccitt / 4096` | `crc` | 178.15x | +| IBM Power10 | `crc16-ccitt / 1048576` | `crc` | 176.67x | +| IBM Power10 | `crc16-ibm / 1048576` | `crc` | 176.60x | +| Intel Sapphire Rapids | `crc16-ccitt / 1048576` | `crc` | 176.46x | +| IBM Power10 | `crc16-ccitt / 262144` | `crc` | 175.61x | ## Top Five Loss Areas -- `argon2id-owasp` / `hash`: 0.97x geomean across 9 rows; W/T/L 4/1/4; pressure `rustcrypto` 4. -- `ed25519` / `verify`: 1.00x geomean across 36 rows; W/T/L 7/20/9; pressure `ring` 6, `dalek` 3. -- `mlkem1024` / `keygen`: 1.00x geomean across 9 rows; W/T/L 6/0/3; pressure `aws-lc-rs` 3. -- `blake2b256` / `streaming`: 1.02x geomean across 27 rows; W/T/L 10/16/1; pressure `rustcrypto` 1. -- `argon2id-small` / `hash`: 1.02x geomean across 27 rows; W/T/L 12/4/11; pressure `rustcrypto` 11. +- `ecdsa-p384` / `sign`: 0.70x geomean across 32 rows; W/T/L 12/0/20; pressure `aws-lc-rs` 16, `rustcrypto-p384` 4. +- `ecdsa-p256` / `verify`: 0.84x geomean across 32 rows; W/T/L 20/7/5; pressure `rustcrypto-p256` 4, `aws-lc-rs` 1. +- `rapidhash-stream` / `one-write`: 0.87x geomean across 88 rows; W/T/L 27/25/36; pressure `rapidhash` 36. +- `ecdsa-p256` / `sign`: 0.91x geomean across 32 rows; W/T/L 28/0/4; pressure `ring` 4. +- `argon2id-owasp` / `hash`: 0.98x geomean across 8 rows; W/T/L 3/1/4; pressure `rustcrypto` 3, `dryoc` 1. ## External Pressure | External | Pairs | W/T/L | Win % | Geomean | Median | | --- | --- | --- | --- | --- | --- | -| `xxhash-rust` | 198 | 62/98/38 | 31% | 1.09x | 1.00x | -| `rapidhash` | 396 | 131/204/61 | 33% | 1.09x | 1.01x | -| `aws-lc-rs` | 1,673 | 1,044/384/245 | 62% | 1.25x | 1.15x | -| `ascon-hash` | 198 | 129/67/2 | 65% | 1.32x | 1.19x | -| `ascon-aead` | 198 | 148/50/0 | 75% | 1.33x | 1.15x | -| `aegis-crate` | 198 | 89/88/21 | 45% | 1.37x | 1.04x | -| `blake3` | 432 | 234/168/30 | 54% | 1.41x | 1.08x | -| `dalek` | 108 | 85/18/5 | 79% | 1.47x | 1.27x | -| `sha2` | 531 | 284/243/4 | 53% | 1.52x | 1.06x | -| `ring` | 1,656 | 1,333/228/95 | 80% | 1.66x | 1.29x | -| `rustcrypto` | 2,745 | 1,949/621/175 | 71% | 1.78x | 1.19x | -| `libcrux` | 81 | 81/0/0 | 100% | 1.79x | 1.70x | -| `dryoc` | 360 | 325/29/6 | 90% | 1.79x | 1.83x | -| `tiny-keccak` | 396 | 267/104/25 | 67% | 1.81x | 1.96x | -| `crc-fast` | 297 | 170/95/32 | 57% | 2.00x | 1.15x | -| `sha3` | 414 | 368/46/0 | 89% | 2.15x | 2.12x | -| `crc64fast` | 99 | 76/12/11 | 77% | 2.39x | 1.95x | -| `crc32fast` | 99 | 81/7/11 | 82% | 2.41x | 2.01x | -| `crc32c` | 99 | 89/6/4 | 90% | 2.71x | 2.19x | -| `fips203` | 81 | 81/0/0 | 100% | 4.66x | 5.11x | -| `rustcrypto-p384` | 72 | 72/0/0 | 100% | 4.79x | 5.26x | -| `rustcrypto-p256` | 72 | 72/0/0 | 100% | 5.65x | 4.91x | -| `rustcrypto-rsa` | 81 | 81/0/0 | 100% | 5.68x | 4.70x | -| `crc` | 297 | 291/2/4 | 98% | 19.45x | 28.96x | +| `rapidhash` | 400 | 168/111/121 | 42% | 1.07x | 1.01x | +| `xxhash-rust` | 272 | 116/89/67 | 43% | 1.08x | 1.00x | +| `aws-lc-rs` | 1,434 | 896/343/195 | 62% | 1.21x | 1.13x | +| `aegis-crate` | 176 | 81/65/30 | 46% | 1.23x | 1.04x | +| `ascon-hash` | 176 | 122/51/3 | 69% | 1.34x | 1.32x | +| `blake3` | 384 | 187/134/63 | 49% | 1.35x | 1.04x | +| `ascon-aead` | 176 | 136/39/1 | 77% | 1.39x | 1.38x | +| `dalek` | 96 | 80/12/4 | 83% | 1.52x | 1.49x | +| `sha2` | 472 | 276/194/2 | 58% | 1.60x | 1.07x | +| `ring` | 1,472 | 1,154/237/81 | 78% | 1.63x | 1.28x | +| `libcrux` | 72 | 72/0/0 | 100% | 1.79x | 1.72x | +| `dryoc` | 320 | 293/22/5 | 92% | 1.81x | 1.85x | +| `rustcrypto` | 2,440 | 1,783/529/128 | 73% | 1.87x | 1.21x | +| `tiny-keccak` | 352 | 237/95/20 | 67% | 1.92x | 2.10x | +| `crc-fast` | 264 | 153/101/10 | 58% | 2.15x | 1.20x | +| `sha3` | 368 | 324/44/0 | 88% | 2.32x | 2.15x | +| `crc32fast` | 88 | 79/4/5 | 90% | 2.75x | 2.06x | +| `crc64fast` | 88 | 73/12/3 | 83% | 2.78x | 2.49x | +| `rustcrypto-p256` | 64 | 56/0/8 | 88% | 3.03x | 3.10x | +| `rustcrypto-p384` | 64 | 56/0/8 | 88% | 3.06x | 5.50x | +| `crc32c` | 88 | 83/3/2 | 94% | 3.13x | 2.29x | +| `fips203` | 72 | 72/0/0 | 100% | 5.28x | 6.07x | +| `rustcrypto-rsa` | 72 | 72/0/0 | 100% | 6.07x | 6.50x | +| `crc` | 264 | 262/0/2 | 99% | 25.76x | 46.98x | ## macOS Local Snapshot -The macOS Apple Silicon run is local evidence from the 2026-07-04 full benchmark at commit `596498f`. It is useful for Apple Silicon planning but is not folded into Linux CI release claims. The ML-KEM row uses the same artifact's public ML-KEM rows. +The macOS Apple Silicon run is local evidence from the 2026-07-04 full benchmark at commit `596498f`, carried forward unchanged in this refresh. It is useful for Apple Silicon planning but is not folded into Linux CI release claims. The ML-KEM row uses the same artifact's public ML-KEM rows. | Scope | Pairs | W/T/L | Win % | Geomean | Median | | --- | --- | --- | --- | --- | --- | @@ -327,29 +344,28 @@ The macOS Apple Silicon run is local evidence from the 2026-07-04 full benchmark ## README Numbers -- **Headline:** 4,052 of 6,750 matched Linux CI fastest-external comparisons are wins; 6,101 are wins or ties. Linux CI geomean is 1.59x. -- **Checksums:** 5.18x geomean across 693 Linux CI fastest-external rows; W/T/L 517/115/61. -- **Hashes/MACs/XOFs:** 1.35x geomean across 3,726 Linux CI fastest-external rows; W/T/L 2,028/1,457/241. -- **Auth/KDF:** 1.25x geomean across 180 Linux CI fastest-external rows; W/T/L 159/20/1. -- **Password hashing:** 1.07x geomean across 135 Linux CI fastest-external rows; W/T/L 69/28/38. -- **Public-key:** 1.33x geomean across 333 Linux CI fastest-external rows; W/T/L 217/73/43. -- **RSA:** 1.55x geomean across 99 Linux CI fastest-external rows; W/T/L 89/2/8. -- **AEAD:** 1.56x geomean across 1,584 Linux CI fastest-external rows; W/T/L 973/354/257. -- **ML-KEM:** 1.49x geomean across 81 Linux CI fastest-external rows; W/T/L 68/4/9. -- **ECDSA P-256/P-384:** 1.45x Linux CI geomean across 144 fastest-external rows; W/T/L 116/7/21. -- **Current top losses:** `argon2id-owasp` / `hash`: 0.97x geomean across 9 rows; W/T/L 4/1/4; pressure `rustcrypto` 4; `ed25519` / `verify`: 1.00x geomean across 36 rows; W/T/L 7/20/9; pressure `ring` 6, `dalek` 3; `mlkem1024` / `keygen`: 1.00x geomean across 9 rows; W/T/L 6/0/3; pressure `aws-lc-rs` 3; `blake2b256` / `streaming`: 1.02x geomean across 27 rows; W/T/L 10/16/1; pressure `rustcrypto` 1; `argon2id-small` / `hash`: 1.02x geomean across 27 rows; W/T/L 12/4/11; pressure `rustcrypto` 11. +- **Headline:** 3,780 of 6,144 matched Linux CI fastest-external comparisons are wins; 5,475 are wins or ties. Linux CI geomean is 1.62x. +- **Checksums:** 6.18x geomean across 616 Linux CI fastest-external rows; W/T/L 476/118/22. +- **Hashes/MACs/XOFs:** 1.35x geomean across 3,456 Linux CI fastest-external rows; W/T/L 1,926/1,181/349. +- **Auth/KDF:** 1.28x geomean across 160 Linux CI fastest-external rows; W/T/L 140/20/0. +- **Password hashing:** 1.07x geomean across 120 Linux CI fastest-external rows; W/T/L 55/27/38. +- **Public-key:** 1.09x geomean across 296 Linux CI fastest-external rows; W/T/L 187/59/50. +- **RSA:** 1.65x geomean across 88 Linux CI fastest-external rows; W/T/L 86/2/0. +- **AEAD:** 1.61x geomean across 1,408 Linux CI fastest-external rows; W/T/L 910/288/210. +- **ML-KEM:** 1.55x geomean across 72 Linux CI fastest-external rows; W/T/L 64/0/8. +- **ECDSA P-256/P-384:** 0.87x Linux CI geomean across 128 fastest-external rows; W/T/L 88/7/33. +- **Current top losses:** `ecdsa-p384` / `sign`: 0.70x geomean across 32 rows; W/T/L 12/0/20; pressure `aws-lc-rs` 16, `rustcrypto-p384` 4; `ecdsa-p256` / `verify`: 0.84x geomean across 32 rows; W/T/L 20/7/5; pressure `rustcrypto-p256` 4, `aws-lc-rs` 1; `rapidhash-stream` / `one-write`: 0.87x geomean across 88 rows; W/T/L 27/25/36; pressure `rapidhash` 36; `ecdsa-p256` / `sign`: 0.91x geomean across 32 rows; W/T/L 28/0/4; pressure `ring` 4; `argon2id-owasp` / `hash`: 0.98x geomean across 8 rows; W/T/L 3/1/4; pressure `rustcrypto` 3, `dryoc` 1. ## Raw Results | Platform | Mode | Date/time | Parsed rows | Result | | --- | --- | --- | --- | --- | -| AMD Zen4 | `ci` | `2026-07-04 15_26_12` | 2,356 | `benchmark_results/2026-07-04/linux/amd-zen4/results.txt` | -| AMD Zen5 | `ci` | `2026-07-04 15_26_12` | 2,356 | `benchmark_results/2026-07-04/linux/amd-zen5/results.txt` | -| AWS Graviton3 | `ci` | `2026-07-04 15_26_12` | 2,367 | `benchmark_results/2026-07-04/linux/graviton3/results.txt` | -| AWS Graviton4 | `ci` | `2026-07-04 15_26_12` | 2,367 | `benchmark_results/2026-07-04/linux/graviton4/results.txt` | -| IBM Power10 | `ci` | `2026-07-04 15_26_12` | 2,107 | `benchmark_results/2026-07-04/linux/ibm-power10/results.txt` | -| IBM z16/s390x | `ci` | `2026-07-04 15_26_12` | 2,107 | `benchmark_results/2026-07-04/linux/ibm-s390x/results.txt` | -| Intel Ice Lake | `ci` | `2026-07-04 15_26_12` | 2,356 | `benchmark_results/2026-07-04/linux/intel-icl/results.txt` | -| Intel Sapphire Rapids | `ci` | `2026-07-04 15_26_12` | 2,356 | `benchmark_results/2026-07-04/linux/intel-spr/results.txt` | -| RISE RISC-V | `ci` | `2026-07-04 15_26_12` | 2,356 | `benchmark_results/2026-07-04/linux/rise-riscv/results.txt` | +| AMD Zen4 | `ci` | `2026-08-18 21_03_07` | 2,304 | `benchmark_results/2026-08-18/linux/amd-zen4/results.txt` | +| AMD Zen5 | `ci` | `2026-08-18 21_03_07` | 2,304 | `benchmark_results/2026-08-18/linux/amd-zen5/results.txt` | +| AWS Graviton3 | `ci` | `2026-08-18 21_03_07` | 2,308 | `benchmark_results/2026-08-18/linux/graviton3/results.txt` | +| AWS Graviton4 | `ci` | `2026-08-18 21_03_07` | 2,308 | `benchmark_results/2026-08-18/linux/graviton4/results.txt` | +| IBM Power10 | `ci` | `2026-08-18 21_03_07` | 2,055 | `benchmark_results/2026-08-18/linux/ibm-power10/results.txt` | +| IBM z16/s390x | `ci` | `2026-08-18 21_03_07` | 2,055 | `benchmark_results/2026-08-18/linux/ibm-s390x/results.txt` | +| Intel Ice Lake | `ci` | `2026-08-18 21_03_07` | 2,304 | `benchmark_results/2026-08-18/linux/intel-icl/results.txt` | +| Intel Sapphire Rapids | `ci` | `2026-08-18 21_03_07` | 2,304 | `benchmark_results/2026-08-18/linux/intel-spr/results.txt` | | macOS Apple Silicon | `local` | `2026-07-04 12_28_04` | 2,277 | `benchmark_results/2026-07-04/macos/aarch64/results.txt` | diff --git a/docs/benchmarking.md b/docs/benchmarking.md index f3235c0e..234e2bb3 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -7,11 +7,12 @@ Benchmark numbers are only meaningful with their platform, commit, feature set, and comparison shape. Treat every headline number as a pointer to the raw results in [`benchmark_results/`](../benchmark_results/). -The published 2026-07-04 aggregate is archival, not an equivalent-work -performance claim. Its RustCrypto HMAC-SHA-256 rows include key setup inside -the timed loop while the rscrypto, `ring`, and AWS-LC rows reuse keyed state. -The current benchmark source corrects that mismatch; publish a new aggregate -only after regenerating the complete artifact. +The published aggregate is the 2026-08-18 eight-runner Linux CI pass at commit +`7eb44e9`. The earlier RustCrypto HMAC-SHA-256 mismatch—key setup inside the +timed loop while the rscrypto, `ring`, and AWS-LC rows reused keyed state—is +corrected in the benchmark source and in this artifact, so the aggregate is an +equivalent-work claim. The RISE RISC-V runner did not execute in that run, so +row counts are not comparable to the nine-runner 2026-07-04 snapshot. ## Read the numbers @@ -26,8 +27,10 @@ Values above `1.00x` mean `rscrypto` was faster for that row. Values below The published W/T/L summaries classify ratios above `1.05x` as wins, ratios from `0.95x` through `1.05x` as ties, and ratios below `0.95x` as losses. Use -individual equivalent-work rows—not the archival aggregate—when a primitive or -message size matters to a deployment. +individual rows—not the crate-wide aggregate—when a primitive or message size +matters to a deployment. A single regressed runner can move a whole category +aggregate: in the 2026-08-18 pass the s390x ECDSA regression pulls the ECDSA +geomean from above parity to `0.87x`. ## Published sources diff --git a/docs/platforms.md b/docs/platforms.md index d672ad9a..fa140ec0 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -66,8 +66,10 @@ Build coverage does not establish a constant-time claim; use ## Per-platform benchmark evidence -The historical 2026-07-04 per-platform results live in -[`benchmark_results/OVERVIEW.md`](../benchmark_results/OVERVIEW.md#coverage-matrix). -Its aggregate includes the equivalent-work limitation documented in -[`benchmarking.md`](benchmarking.md). Benchmark the deployment workload on its -target CPU before choosing a performance-sensitive backend or feature set. +The 2026-08-18 per-platform results live in +[`benchmark_results/OVERVIEW.md`](../benchmark_results/OVERVIEW.md#coverage-matrix), +covering eight Linux CI runners; the RISE RISC-V runner did not execute in that +run. Per-platform results diverge: the same commit measures `2.77x` on IBM +z16/s390x and `1.36x` on AWS Graviton3, and s390x carries a large ECDSA +regression in this snapshot. Benchmark the deployment workload on its target CPU +before choosing a performance-sensitive backend or feature set. diff --git a/scripts/render_perf_chart.rs b/scripts/render_perf_chart.rs index a60efb58..56ccca6a 100644 --- a/scripts/render_perf_chart.rs +++ b/scripts/render_perf_chart.rs @@ -26,6 +26,7 @@ const MUTED: &str = "#7d8590"; const RULE: &str = "#30363d"; const BLUE: &str = "#0090FF"; const TRACK: &str = "#21262d"; +const LOSS: &str = "#FF7B72"; const CHART_TITLE: &str = "rscrypto"; const CHART_SUBTITLE: &str = "Geomean speedups vs fastest matched competitors. Higher is better."; @@ -34,9 +35,10 @@ const SUMMARY_APPLE_LABEL: &str = "Apple Silicon"; const CHECKSUM_TITLE: &str = "Checksums"; const CHECKSUM_COMPETITORS: &str = "- Competitor Crates/Libs: crc-fast, crc, crc32fast, crc32c, crc64fast"; const GROUP_TITLE: &str = "Primitive Geomeans"; +const GROUP_PARITY_NOTE: &str = "1.00x"; const FOOTER_LINUX_RUNNERS: &str = concat!( "- Linux Runners: AMD Zen 4/5; Intel Sapphire Rapids/Ice Lake; ", - "AWS Graviton 3/4; IBM POWER 10 and IBM Z16 (s390x); Rise RISC-V", + "AWS Graviton 3/4; IBM POWER 10 and IBM Z16 (s390x)", ); const FOOTER_MACOS: &str = "- macOS: MBP M1 10-Core, 16GB RAM - Local Dev Box"; const FOOTER_FASTEST_EXTERNAL: &str = @@ -561,22 +563,51 @@ fn render_group_bars(svg: &mut String, data: &ChartData) { let row_y = 398.0; let row_gap = 21.0; let bar_h = 11.0; - let max = 1.60_f64; + let (lo, hi) = group_axis(data); + let span = hi - lo; + let pos = |value: f64| bar_x + ((value.clamp(lo, hi) - lo) / span) * bar_w; + let parity_x = pos(1.0); for (idx, (label, value)) in data.rows.iter().enumerate() { let y = row_y + (idx as f64) * row_gap; - let clamped = value.clamp(1.0, max); - let width = ((clamped - 1.0) / (max - 1.0)) * bar_w; + let value_x_pos = pos(*value); text(svg, label_x, y + 10.0, 13, 700, TEXT, label); svg.push_str(&format!( "" )); + // Bars grow from the 1.00x parity marker: wins extend right, losses extend left. + let (fill_x, fill_w, fill) = if *value >= 1.0 { + (parity_x, value_x_pos - parity_x, BLUE) + } else { + (value_x_pos, parity_x - value_x_pos, LOSS) + }; svg.push_str(&format!( - "" + "" )); mono(svg, value_x, y + 10.0, 15, 850, TITLE, &format!("{value:.2}x")); } + + let rule_top = row_y - 4.0; + let rule_bottom = row_y + (data.rows.len() as f64 - 1.0) * row_gap + bar_h + 4.0; + svg.push_str(&format!( + "" + )); + mono(svg, parity_x - 15.0, rule_bottom + 12.0, 9, 500, MUTED, GROUP_PARITY_NOTE); +} + +/// Axis bounds for the primitive bars: always straddle parity, never clip a bar. +fn group_axis(data: &ChartData) -> (f64, f64) { + let mut lo = 1.0_f64; + let mut hi = 1.0_f64; + for (_, value) in &data.rows { + lo = lo.min(*value); + hi = hi.max(*value); + } + let lo = ((lo - 0.05) * 10.0).floor() / 10.0; + let hi = ((hi + 0.05) * 10.0).ceil() / 10.0; + (lo.min(0.9), hi.max(1.6)) } fn render_evidence_bullets(svg: &mut String) {